From e9a96888bcc6e12dcb4ed8e1d5e1abf85be01e94 Mon Sep 17 00:00:00 2001
From: mhamilton723 <marhamil@microsoft.com>
Date: Wed, 7 Aug 2024 20:51:17 +0000
Subject: [PATCH] Deploy website - based on
 6da5f57401d7032ebd4fdccdba3e3ca5fbac6c83

---
 404.html                                      |  8 +++---
 assets/js/00e0a48e.36781a06.js                |  1 +
 assets/js/120352d6.0e62ec09.js                |  1 -
 assets/js/120352d6.6c87737b.js                |  1 +
 assets/js/935f2afb.0c4b2f48.js                |  1 +
 assets/js/935f2afb.57f575fd.js                |  1 -
 assets/js/dd3f9ada.7a974bf8.js                |  1 -
 assets/js/dd3f9ada.97671d28.js                |  1 +
 assets/js/main.acd258f6.js                    |  2 ++
 ...CENSE.txt => main.acd258f6.js.LICENSE.txt} |  0
 assets/js/main.f50c121d.js                    |  2 --
 assets/js/runtime~main.6b063ea3.js            |  1 +
 assets/js/runtime~main.ea772462.js            |  1 -
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 blog/archive/index.html                       |  8 +++---
 blog/index.html                               |  8 +++---
 blog/overview/index.html                      |  8 +++---
 docs/0.11.3/Deploy Models/Overview/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../Explore Algorithms/OpenAI/index.html      |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/0.11.3/Overview/index.html               |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../Reference/Developer Setup/index.html      |  8 +++---
 docs/0.11.3/Reference/Docker Setup/index.html |  8 +++---
 docs/0.11.3/Reference/Dotnet Setup/index.html |  8 +++---
 .../index.html                                |  8 +++---
 docs/0.11.3/Reference/R Setup/index.html      |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 .../0.11.3/Use with MLFlow/Install/index.html |  8 +++---
 .../Use with MLFlow/Overview/index.html       |  8 +++---
 docs/0.11.4/Deploy Models/Overview/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../Explore Algorithms/OpenAI/index.html      |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/0.11.4/Overview/index.html               |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../Reference/Developer Setup/index.html      |  8 +++---
 docs/0.11.4/Reference/Docker Setup/index.html |  8 +++---
 docs/0.11.4/Reference/Dotnet Setup/index.html |  8 +++---
 .../index.html                                |  8 +++---
 docs/0.11.4/Reference/R Setup/index.html      |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 .../0.11.4/Use with MLFlow/Install/index.html |  8 +++---
 .../Use with MLFlow/Overview/index.html       |  8 +++---
 docs/1.0.1/Deploy Models/Overview/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../Explore Algorithms/OpenAI/index.html      |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/1.0.1/Overview/index.html                |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../Reference/Developer Setup/index.html      |  8 +++---
 docs/1.0.1/Reference/Docker Setup/index.html  |  8 +++---
 docs/1.0.1/Reference/Dotnet Setup/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 docs/1.0.1/Reference/R Setup/index.html       |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 docs/1.0.1/Use with MLFlow/Install/index.html |  8 +++---
 .../1.0.1/Use with MLFlow/Overview/index.html |  8 +++---
 docs/1.0.2/Deploy Models/Overview/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../Explore Algorithms/OpenAI/index.html      |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/1.0.2/Overview/index.html                |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../Reference/Developer Setup/index.html      |  8 +++---
 docs/1.0.2/Reference/Docker Setup/index.html  |  8 +++---
 docs/1.0.2/Reference/Dotnet Setup/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 docs/1.0.2/Reference/R Setup/index.html       |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 docs/1.0.2/Use with MLFlow/Install/index.html |  8 +++---
 .../1.0.2/Use with MLFlow/Overview/index.html |  8 +++---
 docs/1.0.3/Deploy Models/Overview/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../Explore Algorithms/OpenAI/index.html      |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/1.0.3/Overview/index.html                |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../Reference/Developer Setup/index.html      |  8 +++---
 docs/1.0.3/Reference/Docker Setup/index.html  |  8 +++---
 docs/1.0.3/Reference/Dotnet Setup/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 docs/1.0.3/Reference/R Setup/index.html       |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 docs/1.0.3/Use with MLFlow/Install/index.html |  8 +++---
 .../1.0.3/Use with MLFlow/Overview/index.html |  8 +++---
 docs/Deploy Models/Overview/index.html        |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - OpenAI Embedding/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 docs/Explore Algorithms/OpenAI/index.html     |  8 +++---
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 docs/Get Started/Install SynapseML/index.html |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/Overview/index.html                      |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 docs/Reference/Contributor Guide/index.html   |  8 +++---
 docs/Reference/Developer Setup/index.html     |  8 +++---
 docs/Reference/Docker Setup/index.html        |  8 +++---
 docs/Reference/Dotnet Setup/index.html        |  8 +++---
 .../index.html                                |  8 +++---
 docs/Reference/R Setup/index.html             |  8 +++---
 docs/Use with MLFlow/Autologging/index.html   |  8 +++---
 docs/Use with MLFlow/Install/index.html       |  8 +++---
 docs/Use with MLFlow/Overview/index.html      |  8 +++---
 docs/next/Deploy Models/Overview/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Geospatial Services/index.html            |  8 +++---
 .../Multivariate Anomaly Detection/index.html |  8 +++---
 .../AI Services/Overview/index.html           |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Analyze Text/index.html      |  8 +++---
 .../Quickstart - Create Audiobooks/index.html |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Flooding Risk/index.html     |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Isolation Forests/index.html |  8 +++---
 .../Causal Inference/Overview/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Quickstart - Train Classifier/index.html  |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Distributed Training/index.html           |  8 +++---
 .../Deep Learning/Getting Started/index.html  |  8 +++---
 .../Deep Learning/ONNX/index.html             |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Hyperparameter Tuning/HyperOpt/index.html |  8 +++---
 .../Quickstart - Random Search/index.html     |  8 +++---
 .../LightGBM/Overview/index.html              |  8 +++---
 .../index.html                                |  8 +++---
 .../OpenAI/Langchain/index.html               | 10 +++----
 .../index.html                                | 28 +++++++++++++++++++
 .../index.html                                | 12 ++++----
 .../Quickstart - OpenAI Embedding/index.html  | 10 +++----
 .../index.html                                | 10 +++----
 .../next/Explore Algorithms/OpenAI/index.html | 10 +++----
 .../OpenCV/Image Transformations/index.html   |  8 +++---
 .../Other Algorithms/Cyber ML/index.html      |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Smart Adaptive Recommendations/index.html |  8 +++---
 .../Quickstart - Data Cleaning/index.html     |  8 +++---
 .../Quickstart - Train Regressor/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 .../Data Balance Analysis/index.html          |  8 +++---
 .../Explanation Dashboard/index.html          |  8 +++---
 .../Image Explainers/index.html               |  8 +++---
 .../Interpreting Model Predictions/index.html |  8 +++---
 .../PDP and ICE Explainers/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Tabular Explainers/index.html             |  8 +++---
 .../Responsible AI/Text Explainers/index.html |  8 +++---
 .../Contextual Bandits/index.html             |  8 +++---
 .../Multi-class classification/index.html     |  8 +++---
 .../Vowpal Wabbit/Overview/index.html         |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../index.html                                |  8 +++---
 .../Create a Spark Cluster/index.html         |  8 +++---
 .../Get Started/Install SynapseML/index.html  |  8 +++---
 .../Quickstart - Your First Models/index.html |  8 +++---
 .../Set up Cognitive Services/index.html      |  8 +++---
 docs/next/Overview/index.html                 |  8 +++---
 .../estimators/estimators_causal/index.html   |  8 +++---
 .../estimators_cognitive/index.html           |  8 +++---
 .../estimators/estimators_core/index.html     |  8 +++---
 .../estimators/estimators_lightgbm/index.html |  8 +++---
 .../estimators/estimators_vw/index.html       |  8 +++---
 .../transformers_cognitive/index.html         |  8 +++---
 .../transformers/transformers_core/index.html |  8 +++---
 .../transformers_deep_learning/index.html     |  8 +++---
 .../transformers_opencv/index.html            |  8 +++---
 .../transformers/transformers_vw/index.html   |  8 +++---
 .../Reference/Contributor Guide/index.html    |  8 +++---
 .../next/Reference/Developer Setup/index.html |  8 +++---
 docs/next/Reference/Docker Setup/index.html   |  8 +++---
 docs/next/Reference/Dotnet Setup/index.html   |  8 +++---
 .../index.html                                |  8 +++---
 docs/next/Reference/R Setup/index.html        |  8 +++---
 .../Use with MLFlow/Autologging/index.html    |  8 +++---
 docs/next/Use with MLFlow/Install/index.html  |  8 +++---
 docs/next/Use with MLFlow/Overview/index.html |  8 +++---
 index.html                                    |  8 +++---
 search/index.html                             |  8 +++---
 sitemap.xml                                   |  2 +-
 videos/index.html                             |  8 +++---
 611 files changed, 2430 insertions(+), 2401 deletions(-)
 create mode 100644 assets/js/00e0a48e.36781a06.js
 delete mode 100644 assets/js/120352d6.0e62ec09.js
 create mode 100644 assets/js/120352d6.6c87737b.js
 create mode 100644 assets/js/935f2afb.0c4b2f48.js
 delete mode 100644 assets/js/935f2afb.57f575fd.js
 delete mode 100644 assets/js/dd3f9ada.7a974bf8.js
 create mode 100644 assets/js/dd3f9ada.97671d28.js
 create mode 100644 assets/js/main.acd258f6.js
 rename assets/js/{main.f50c121d.js.LICENSE.txt => main.acd258f6.js.LICENSE.txt} (100%)
 delete mode 100644 assets/js/main.f50c121d.js
 create mode 100644 assets/js/runtime~main.6b063ea3.js
 delete mode 100644 assets/js/runtime~main.ea772462.js
 create mode 100644 docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/index.html

diff --git a/404.html b/404.html
index b14d371a32..0ca258abb3 100644
--- a/404.html
+++ b/404.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/404" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--xl"><div class="row"><div class="col col--6 col--offset-3 notFoundContainer_NXNv"><h1 class="hero__title">Sorry! Page Not Found</h1><p>We have been doing some work on our website, chances are that the page you&#x27;re looking for is <a href="./docs/Overview/">in the new docs section</a>.</p></div></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/assets/js/00e0a48e.36781a06.js b/assets/js/00e0a48e.36781a06.js
new file mode 100644
index 0000000000..822e5ac087
--- /dev/null
+++ b/assets/js/00e0a48e.36781a06.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[96922],{3905:(e,t,n)=>{n.d(t,{Zo:()=>p,kt:()=>u});var r=n(67294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function s(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?s(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):s(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function o(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},s=Object.keys(e);for(r=0;r<s.length;r++)n=s[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var s=Object.getOwnPropertySymbols(e);for(r=0;r<s.length;r++)n=s[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var d=r.createContext({}),l=function(e){var t=r.useContext(d),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},p=function(e){var t=l(e.components);return r.createElement(d.Provider,{value:t},e.children)},c={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},m=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,s=e.originalType,d=e.parentName,p=o(e,["components","mdxType","originalType","parentName"]),m=l(n),u=a,g=m["".concat(d,".").concat(u)]||m[u]||c[u]||s;return n?r.createElement(g,i(i({ref:t},p),{},{components:n})):r.createElement(g,i({ref:t},p))}));function u(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var s=n.length,i=new Array(s);i[0]=m;var o={};for(var d in t)hasOwnProperty.call(t,d)&&(o[d]=t[d]);o.originalType=e,o.mdxType="string"==typeof e?e:a,i[1]=o;for(var l=2;l<s;l++)i[l]=n[l];return r.createElement.apply(null,i)}return r.createElement.apply(null,n)}m.displayName="MDXCreateElement"},7174:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>d,contentTitle:()=>i,default:()=>c,frontMatter:()=>s,metadata:()=>o,toc:()=>l});var r=n(83117),a=(n(67294),n(3905));const s={title:"Quickstart - Custom Embeddings and Approximate KNN on GPU",hide_title:!0,status:"stable"},i="Embedding Text with local (per node) NVIDIA TensorRT accelerator and GPU based Aproximate Nearest Neighbor (ANN)",o={unversionedId:"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU",id:"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU",title:"Quickstart - Custom Embeddings and Approximate KNN on GPU",description:"The demo extending existing Azure OpenAI based demo when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using multilingual E5 text embeddings and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using NVIDIA TensorRT and Spark Rapids ML. The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to previos CPU based demo",source:"@site/docs/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU.md",sourceDirName:"Explore Algorithms/OpenAI",slug:"/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU",draft:!1,tags:[],version:"current",frontMatter:{title:"Quickstart - Custom Embeddings and Approximate KNN on GPU",hide_title:!0,status:"stable"},sidebar:"docs",previous:{title:"Quickstart - OpenAI Embedding and GPU based KNN",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN"},next:{title:"Quickstart - Understand and Search Forms",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms"}},d={},l=[{value:"Step 1: Prepare Environment",id:"step-1-prepare-environment",level:2},{value:"Step 2: Load Input Data",id:"step-2-load-input-data",level:2},{value:"Step 3: Generate Embeddings",id:"step-3-generate-embeddings",level:2},{value:"Step 4: Build the query against embeddings",id:"step-4-build-the-query-against-embeddings",level:2},{value:"Step 5: Build a fast vector index to over review embeddings",id:"step-5-build-a-fast-vector-index-to-over-review-embeddings",level:2},{value:"Step 6: Find top k Nearest Neighbors ON GPU",id:"step-6-find-top-k-nearest-neighbors-on-gpu",level:2},{value:"Step 7: Collect and display results",id:"step-7-collect-and-display-results",level:2}],p={toc:l};function c(e){let{components:t,...n}=e;return(0,a.kt)("wrapper",(0,r.Z)({},p,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"embedding-text-with-local-per-node-nvidia-tensorrt-accelerator-and-gpu-based-aproximate-nearest-neighbor-ann"},"Embedding Text with local (per node) NVIDIA TensorRT accelerator and GPU based Aproximate Nearest Neighbor (ANN)"),(0,a.kt)("p",null,"The demo extending existing ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN.ipynb"},"Azure OpenAI based demo")," when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using ",(0,a.kt)("a",{parentName:"p",href:"https://arxiv.org/abs/2402.05672"},"multilingual E5 text embeddings")," and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using ",(0,a.kt)("a",{parentName:"p",href:"https://developer.nvidia.com/tensorrt"},"NVIDIA TensorRT")," and ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/NVIDIA/spark-rapids-ml"},"Spark Rapids ML"),". The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding.ipynb"},"previos CPU based demo")),(0,a.kt)("p",null,"The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but currently the notebook was running on Databricks GPU based cluster using Standard_NC24ads_A100_v4 with 6 workers. Databricks Runtime was 13.3 LTS ML (includes Apache Spark 3.4.1, GPU, Scala 2.12) with related ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/microsoft/SynapseML/tree/master/tools/init_scripts"},"init_script")," to install all required packages."),(0,a.kt)("h2",{id:"step-1-prepare-environment"},"Step 1: Prepare Environment"),(0,a.kt)("p",null,"It will imports required libraries and get initial settings"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"import torch\nimport sys\nimport pyspark.sql.functions as F\nfrom pyspark.sql.types import StructType, StructField, IntegerType, StringType\nfrom pyspark.ml.linalg import Vectors\nfrom pyspark.ml.linalg import VectorUDT\nfrom spark_rapids_ml.knn import (\n    ApproximateNearestNeighbors,\n    ApproximateNearestNeighborsModel,\n)\nfrom synapse.ml.hf import HuggingFaceSentenceEmbedder\nfrom synapse.ml.nn import KNN\n")),(0,a.kt)("h2",{id:"step-2-load-input-data"},"Step 2: Load Input Data"),(0,a.kt)("p",null,"It will load public dataset and generate extra syntetic rows if set by size parameter"),(0,a.kt)("p",null,"The loaded dataset has 1000 rows. If you specify ",(0,a.kt)("i",null,"number_of_input_rows")," in ","[1..1000]"," it will cut extra rows if needed"),(0,a.kt)("p",null,"If ",(0,a.kt)("i",null,"number_of_input_rows")," in ","[1000..1000000]"," it will generate extra rows using cross join of original data"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'file_path = "wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv"\n\ndf = spark.read.options(inferSchema="True", delimiter=",", header=True).csv(file_path)\ndf = df.withColumn(\n    "data",\n    F.format_string("Title: %s; Content: %s", F.trim(df.Summary), F.trim(df.Text)),\n)\n\n# Size of DF\nnumber_of_input_rows = 100\n\n\n# Check if the row count is less than 10\nif number_of_input_rows <= 0 or number_of_input_rows >= 1000000:\n    raise ValueError(f"Limit is {number_of_input_rows}, which should be less than 1M.")\n\nif number_of_input_rows > 1000:\n\n    # Cross-join the DataFrame with itself to create n x n pairs for string concatenation (synthetic data)\n    cross_joined_df = df.crossJoin(df.withColumnRenamed("data", "data_"))\n\n    # Create a new column \'result_vector\' by concatenating the two source vectors\n    tmp_df = cross_joined_df.withColumn(\n        "result_vector",\n        F.concat(F.col("data"), F.lit(". \\n"), F.col("data_")),\n    )\n\n    # Select only the necessary columns and show the result\n    tmp_df = tmp_df.select("result_vector")\n\n    # Shuffle the DataFrame with a fixed seed to have close strings spreaded\n    seed = 42\n\n    df = (\n        tmp_df.withColumnRenamed("result_vector", "data")\n        .withColumn("id", F.monotonically_increasing_id())\n        .orderBy(F.rand(seed))\n    )\n\ndf = df.limit(number_of_input_rows).repartition(10).cache()\n\nprint(f"Loaded: {number_of_input_rows} rows")\n')),(0,a.kt)("h2",{id:"step-3-generate-embeddings"},"Step 3: Generate Embeddings"),(0,a.kt)("p",null,'We will first generate embeddings using NVIDIA TensorRT optimized SentenceTransformer. In the demo you can use two fifferent HF models: intfloat/e5-large-v2 or sentence-transformers/all-MiniLM-L6-v2"'),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'# To create embedder with different models, uncomment the following line\n# embedder = HuggingFaceSentenceEmbedder(modelName="intfloat/e5-large-v2", inputCol="data", outputCol="embeddings", runtime="tensorrt")\nembedder = HuggingFaceSentenceEmbedder(\n    modelName="sentence-transformers/all-MiniLM-L6-v2",\n    inputCol="data",\n    outputCol="embeddings",\n    runtime="tensorrt",\n)\n\nembeddings = embedder.transform(df).select("id", "embeddings").cache()\n')),(0,a.kt)("h2",{id:"step-4-build-the-query-against-embeddings"},"Step 4: Build the query against embeddings"),(0,a.kt)("p",null,"Get query embeddings running standard SentenceTransformer just on the driver. Convert embedding results to a data frame"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'# Sample query\nqueries = ["desserts", "disgusting"]\nids = [1, 2]\n\n# Create DataFrame directly from the data and schema\nquery_df = spark.createDataFrame(\n    list(zip(ids, queries)),\n    StructType(\n        [\n            StructField("id", IntegerType(), nullable=False),\n            StructField("data", StringType(), nullable=False),\n        ]\n    ),\n)\n\nquery_embeddings = embedder.transform(query_df).select("id", "embeddings").cache()\n')),(0,a.kt)("h2",{id:"step-5-build-a-fast-vector-index-to-over-review-embeddings"},"Step 5: Build a fast vector index to over review embeddings"),(0,a.kt)("p",null,"We will use fast NVIDIA Rapids indexer. This KNN implementation will work only on GPU. If you want to use CPU then switch to synapse.ml.nn CPU based KNN implementation"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"RUN_ON_GPU = torch.cuda.is_available()\n")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'if RUN_ON_GPU:\n    rapids_knn_model = (\n        ApproximateNearestNeighbors(k=5)\n        .setInputCol("embeddings")\n        .setIdCol("id")\n        .fit(embeddings)\n    )\nelse:\n    array_to_vector_udf = udf(lambda array: Vectors.dense(array), VectorUDT())\n    df_with_vectors = embeddings.withColumn(\n        "features", array_to_vector_udf(embeddings["embeddings"])\n    )\n    knn = (\n        KNN()\n        .setFeaturesCol("features")\n        .setValuesCol("id")\n        .setOutputCol("output")\n        .setK(10)\n    )\n    knn_model = knn.fit(df_with_vectors)\n')),(0,a.kt)("h2",{id:"step-6-find-top-k-nearest-neighbors-on-gpu"},"Step 6: Find top k Nearest Neighbors ON GPU"),(0,a.kt)("p",null,"We will use fast ANN ",(0,a.kt)("a",{parentName:"p",href:"https://developer.nvidia.com/blog/accelerated-vector-search-approximating-with-rapids-raft-ivf-flat/"},"IVFFlat algorithm")," from Rapids"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'if RUN_ON_GPU:\n    (_, _, knn_df) = rapids_knn_model.kneighbors(query_embeddings)\nelse:\n    array_to_vector_udf = udf(lambda array: Vectors.dense(array), VectorUDT())\n    df_with_vectors = query_embeddings.withColumn(\n        "features", array_to_vector_udf(query_embeddings["embeddings"])\n    )\n    knn_df = knn_model.transform(df_with_vectors)\n')),(0,a.kt)("h2",{id:"step-7-collect-and-display-results"},"Step 7: Collect and display results"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'if RUN_ON_GPU:\n    result_df = (\n        knn_df.withColumn(\n            "zipped", F.explode(F.arrays_zip(F.col("indices"), F.col("distances")))\n        )\n        .select(\n            F.col("query_id"),\n            F.col("zipped.indices").alias("id"),\n            F.col("zipped.distances").alias("distance"),\n        )\n        .join(df, on="id", how="inner")\n        .select("query_id", "id", "data", "distance")\n    )\nelse:\n    knn_df = knn_df.withColumnRenamed("data", "original_data")\n    result_df = (\n        knn_df.withColumn("match", F.explode("output"))\n        .join(df, df["id"] == F.col("match.value"))\n        .select("original_data", F.col("data"), "match.distance")\n    )\n\ndisplay(result_df)\n')),(0,a.kt)("h1",{id:"results"},"Results"),(0,a.kt)("p",null,"The goal of this demo is to showcase two acceleration techniques: local (per node) embedding generation and approximate KNN. Compared to the original method, which relies on HTTP requests to the OpenAI model and CPU-based KNN. The new approach is significantly more scalable and provides substantial acceleration, especially for large input datasets."),(0,a.kt)("p",null,"This is the comparison dureation results on 10 T4 GPU nodes for both approaches:"),(0,a.kt)("p",null,(0,a.kt)("img",{parentName:"p",src:"https://mmlspark.blob.core.windows.net/graphics/Documentation/knn_comparison.png",alt:"KNN Comparison"})))}c.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/120352d6.0e62ec09.js b/assets/js/120352d6.0e62ec09.js
deleted file mode 100644
index cbafdd864c..0000000000
--- a/assets/js/120352d6.0e62ec09.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[89262],{3905:(e,n,t)=>{t.d(n,{Zo:()=>c,kt:()=>m});var r=t(67294);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function s(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?o(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):o(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,a=function(e,n){if(null==e)return{};var t,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)t=o[r],n.indexOf(t)>=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)t=o[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var l=r.createContext({}),p=function(e){var n=r.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):s(s({},n),e)),t},c=function(e){var n=p(e.components);return r.createElement(l.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),d=p(t),m=a,h=d["".concat(l,".").concat(m)]||d[m]||u[m]||o;return t?r.createElement(h,s(s({ref:n},c),{},{components:t})):r.createElement(h,s({ref:n},c))}));function m(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var o=t.length,s=new Array(o);s[0]=d;var i={};for(var l in n)hasOwnProperty.call(n,l)&&(i[l]=n[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,s[1]=i;for(var p=2;p<o;p++)s[p]=t[p];return r.createElement.apply(null,s)}return r.createElement.apply(null,t)}d.displayName="MDXCreateElement"},17608:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>s,default:()=>u,frontMatter:()=>o,metadata:()=>i,toc:()=>p});var r=t(83117),a=(t(67294),t(3905));const o={title:"Quickstart - Understand and Search Forms",hide_title:!0,status:"stable"},s="Tutorial: Create a custom search engine and question-answering system",i={unversionedId:"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",id:"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",title:"Quickstart - Understand and Search Forms",description:"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:",source:"@site/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",sourceDirName:"Explore Algorithms/OpenAI",slug:"/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",draft:!1,tags:[],version:"current",frontMatter:{title:"Quickstart - Understand and Search Forms",hide_title:!0,status:"stable"},sidebar:"docs",previous:{title:"Quickstart - OpenAI Embedding and GPU based KNN",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN"},next:{title:"Getting Started",permalink:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started"}},l={},p=[{value:"1 - Set up dependencies",id:"1---set-up-dependencies",level:2},{value:"2 - Load data into Spark",id:"2---load-data-into-spark",level:2},{value:"3 - Apply form recognition",id:"3---apply-form-recognition",level:2},{value:"4 - Simplify form recognition output",id:"4---simplify-form-recognition-output",level:2},{value:"5 - Add translations",id:"5---add-translations",level:2},{value:"6 - Translate products to emojis with OpenAI \ud83e\udd2f",id:"6---translate-products-to-emojis-with-openai-",level:2},{value:"7 - Infer vendor address continent with OpenAI",id:"7---infer-vendor-address-continent-with-openai",level:2},{value:"8 - Create an Azure Search Index for the Forms",id:"8---create-an-azure-search-index-for-the-forms",level:2},{value:"9 - Try out a search query",id:"9---try-out-a-search-query",level:2},{value:"10 - Build a chatbot that can use Azure Search as a tool \ud83e\udde0\ud83d\udd27",id:"10---build-a-chatbot-that-can-use-azure-search-as-a-tool-",level:2},{value:"11 - Asking our chatbot a question",id:"11---asking-our-chatbot-a-question",level:2},{value:"12 - A quick double check",id:"12---a-quick-double-check",level:2}],c={toc:p};function u(e){let{components:n,...t}=e;return(0,a.kt)("wrapper",(0,r.Z)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"tutorial-create-a-custom-search-engine-and-question-answering-system"},"Tutorial: Create a custom search engine and question-answering system"),(0,a.kt)("p",null,"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:"),(0,a.kt)("blockquote",null,(0,a.kt)("ul",{parentName:"blockquote"},(0,a.kt)("li",{parentName:"ul"},"Load various forms (invoices) into a data frame in an Apache Spark session"),(0,a.kt)("li",{parentName:"ul"},"Analyze them to determine their features"),(0,a.kt)("li",{parentName:"ul"},"Assemble the resulting output into a tabular data structure"),(0,a.kt)("li",{parentName:"ul"},"Write the output to a search index hosted in Azure Cognitive Search"),(0,a.kt)("li",{parentName:"ul"},"Explore and query over the content you created"))),(0,a.kt)("h2",{id:"1---set-up-dependencies"},"1 - Set up dependencies"),(0,a.kt)("p",null,"We start by importing packages and connecting to the Azure resources used in this workflow."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"%pip install openai==0.28.1\n")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.core.platform import find_secret\n\ncognitive_key = find_secret(\n    secret_name="ai-services-api-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string. e.g. cognitive_key="27snaiw..."\ncognitive_location = "eastus"\n\ntranslator_key = find_secret(\n    secret_name="translator-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\ntranslator_location = "eastus"\n\nsearch_key = find_secret(\n    secret_name="azure-search-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\nsearch_service = "mmlspark-azure-search"\nsearch_index = "form-demo-index-5"\n\nopenai_key = find_secret(\n    secret_name="openai-api-key-2", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\nopenai_service_name = "synapseml-openai-2"\nopenai_deployment_name = "gpt-35-turbo"\nopenai_url = f"https://{openai_service_name}.openai.azure.com/"\n')),(0,a.kt)("h2",{id:"2---load-data-into-spark"},"2 - Load data into Spark"),(0,a.kt)("p",null,"This code loads a few external files from an Azure storage account that's used for demo purposes. The files are various invoices, and they're read into a data frame."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.sql.functions import udf\nfrom pyspark.sql.types import StringType\n\n\ndef blob_to_url(blob):\n    [prefix, postfix] = blob.split("@")\n    container = prefix.split("/")[-1]\n    split_postfix = postfix.split("/")\n    account = split_postfix[0]\n    filepath = "/".join(split_postfix[1:])\n    return "https://{}/{}/{}".format(account, container, filepath)\n\n\ndf2 = (\n    spark.read.format("binaryFile")\n    .load("wasbs://publicwasb@mmlspark.blob.core.windows.net/form_subset/*")\n    .select("path")\n    .limit(10)\n    .select(udf(blob_to_url, StringType())("path").alias("url"))\n    .cache()\n)\n\ndisplay(df2)\n')),(0,a.kt)("img",{src:"https://mmlspark.blob.core.windows.net/graphics/Invoice11205.svg",width:"40%"}),(0,a.kt)("h2",{id:"3---apply-form-recognition"},"3 - Apply form recognition"),(0,a.kt)("p",null,"This code loads the ",(0,a.kt)("a",{parentName:"p",href:"https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#form-recognizer"},"AnalyzeInvoices transformer")," and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.form import AnalyzeInvoices\n\nanalyzed_df = (\n    AnalyzeInvoices()\n    .setSubscriptionKey(cognitive_key)\n    .setLocation(cognitive_location)\n    .setImageUrlCol("url")\n    .setOutputCol("invoices")\n    .setErrorCol("errors")\n    .setConcurrency(5)\n    .transform(df2)\n    .cache()\n)\n\ndisplay(analyzed_df)\n')),(0,a.kt)("h2",{id:"4---simplify-form-recognition-output"},"4 - Simplify form recognition output"),(0,a.kt)("p",null,"This code uses the ",(0,a.kt)("a",{parentName:"p",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.FormOntologyTransformer"},"FormOntologyLearner"),", a transformer that analyzes the output of Form Recognizer transformers (for Azure AI Document Intelligence) and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content."),(0,a.kt)("p",null,"FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.form import FormOntologyLearner\n\norganized_df = (\n    FormOntologyLearner()\n    .setInputCol("invoices")\n    .setOutputCol("extracted")\n    .fit(analyzed_df)\n    .transform(analyzed_df)\n    .select("url", "extracted.*")\n    .cache()\n)\n\ndisplay(organized_df)\n')),(0,a.kt)("p",null,"With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.sql.functions import explode, col\n\nitemized_df = (\n    organized_df.select("*", explode(col("Items")).alias("Item"))\n    .drop("Items")\n    .select("Item.*", "*")\n    .drop("Item")\n)\n\ndisplay(itemized_df)\n')),(0,a.kt)("h2",{id:"5---add-translations"},"5 - Add translations"),(0,a.kt)("p",null,"This code loads ",(0,a.kt)("a",{parentName:"p",href:"https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#translation"},"Translate"),', a transformer that calls the Azure AI Translator service in Azure AI services. The original text, which is in English in the "Description" column, is machine-translated into various languages. All of the output is consolidated into "output.translations" array.'),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.translate import Translate\n\ntranslated_df = (\n    Translate()\n    .setSubscriptionKey(translator_key)\n    .setLocation(translator_location)\n    .setTextCol("Description")\n    .setErrorCol("TranslationError")\n    .setOutputCol("output")\n    .setToLanguage(["zh-Hans", "fr", "ru", "cy"])\n    .setConcurrency(5)\n    .transform(itemized_df)\n    .withColumn("Translations", col("output.translations")[0])\n    .drop("output", "TranslationError")\n    .cache()\n)\n\ndisplay(translated_df)\n')),(0,a.kt)("h2",{id:"6---translate-products-to-emojis-with-openai-"},"6 - Translate products to emojis with OpenAI \ud83e\udd2f"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.openai import OpenAIPrompt\nfrom pyspark.sql.functions import trim, split\n\nemoji_template = """ \n  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma\n  \n  Two Ducks: \ud83e\udd86\ud83e\udd86,\n  Light Bulb: \ud83d\udca1,\n  Three Peaches: \ud83c\udf51\ud83c\udf51\ud83c\udf51,\n  Two kitchen stoves: \u2668\ufe0f\u2668\ufe0f,\n  A red car: \ud83d\ude97,\n  A person and a cat: \ud83e\uddcd\ud83d\udc08,\n  A {Description}: """\n\nprompter = (\n    OpenAIPrompt()\n    .setSubscriptionKey(openai_key)\n    .setDeploymentName(openai_deployment_name)\n    .setUrl(openai_url)\n    .setMaxTokens(5)\n    .setPromptTemplate(emoji_template)\n    .setErrorCol("error")\n    .setOutputCol("Emoji")\n)\n\nemoji_df = (\n    prompter.transform(translated_df)\n    .withColumn("Emoji", trim(split(col("Emoji"), ",").getItem(0)))\n    .drop("error", "prompt")\n    .cache()\n)\n')),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(emoji_df.select("Description", "Emoji"))\n')),(0,a.kt)("h2",{id:"7---infer-vendor-address-continent-with-openai"},"7 - Infer vendor address continent with OpenAI"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'continent_template = """\nWhich continent does the following address belong to? \n\nPick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. \n\nDont respond with anything but one of the above. If you don\'t know the answer or cannot figure it out from the text, return None. End your answer with a comma.\n\nAddress: "6693 Ryan Rd, North Whales",\nContinent: Europe,\nAddress: "6693 Ryan Rd",\nContinent: None,\nAddress: "{VendorAddress}",\nContinent:"""\n\ncontinent_df = (\n    prompter.setOutputCol("Continent")\n    .setPromptTemplate(continent_template)\n    .transform(emoji_df)\n    .withColumn("Continent", trim(split(col("Continent"), ",").getItem(0)))\n    .drop("error", "prompt")\n    .cache()\n)\n')),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(continent_df.select("VendorAddress", "Continent"))\n')),(0,a.kt)("h2",{id:"8---create-an-azure-search-index-for-the-forms"},"8 - Create an Azure Search Index for the Forms"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services import *\nfrom pyspark.sql.functions import monotonically_increasing_id, lit\n\n(\n    continent_df.withColumn("DocID", monotonically_increasing_id().cast("string"))\n    .withColumn("SearchAction", lit("upload"))\n    .writeToAzureSearch(\n        subscriptionKey=search_key,\n        actionCol="SearchAction",\n        serviceName=search_service,\n        indexName=search_index,\n        keyCol="DocID",\n    )\n)\n')),(0,a.kt)("h2",{id:"9---try-out-a-search-query"},"9 - Try out a search query"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import requests\n\nsearch_url = "https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06".format(\n    search_service, search_index\n)\nrequests.post(\n    search_url, json={"search": "door"}, headers={"api-key": search_key}\n).json()\n')),(0,a.kt)("h2",{id:"10---build-a-chatbot-that-can-use-azure-search-as-a-tool-"},"10 - Build a chatbot that can use Azure Search as a tool \ud83e\udde0\ud83d\udd27"),(0,a.kt)("img",{src:"https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg",width:"40%"}),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import json\nimport openai\n\nopenai.api_type = "azure"\nopenai.api_base = openai_url\nopenai.api_key = openai_key\nopenai.api_version = "2023-03-15-preview"\n\nchat_context_prompt = f"""\nYou are a chatbot designed to answer questions with the help of a search engine that has the following information:\n\n{continent_df.columns}\n\nIf you dont know the answer to a question say "I dont know". Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{"query": "example_query"}}\n"""\n\n\ndef search_query_prompt(question):\n    return f"""\nGiven the search engine above, what would you search for to answer the following question?\n\nQuestion: "{question}"\n\nPlease output a json in the form of {{"query": "example_query"}}\n"""\n\n\ndef search_result_prompt(query):\n    search_results = requests.post(\n        search_url, json={"search": query}, headers={"api-key": search_key}\n    ).json()\n    return f"""\n\nYou previously ran a search for "{query}" which returned the following results:\n\n{search_results}\n\nYou should use the results to help you answer questions. If you dont know the answer to a question say "I dont know". Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. \n"""\n\n\ndef prompt_gpt(messages):\n    response = openai.ChatCompletion.create(\n        engine=openai_deployment_name, messages=messages, max_tokens=None, top_p=0.95\n    )\n    return response["choices"][0]["message"]["content"]\n\n\ndef custom_chatbot(question):\n    while True:\n        try:\n            query = json.loads(\n                prompt_gpt(\n                    [\n                        {"role": "system", "content": chat_context_prompt},\n                        {"role": "user", "content": search_query_prompt(question)},\n                    ]\n                )\n            )["query"]\n\n            return prompt_gpt(\n                [\n                    {"role": "system", "content": chat_context_prompt},\n                    {"role": "system", "content": search_result_prompt(query)},\n                    {"role": "user", "content": question},\n                ]\n            )\n        except Exception as e:\n            raise e\n')),(0,a.kt)("h2",{id:"11---asking-our-chatbot-a-question"},"11 - Asking our chatbot a question"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'custom_chatbot("What did Luke Diaz buy?")\n')),(0,a.kt)("h2",{id:"12---a-quick-double-check"},"12 - A quick double check"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(\n    continent_df.where(col("CustomerName") == "Luke Diaz")\n    .select("Description")\n    .distinct()\n)\n')))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/120352d6.6c87737b.js b/assets/js/120352d6.6c87737b.js
new file mode 100644
index 0000000000..dd49e30a58
--- /dev/null
+++ b/assets/js/120352d6.6c87737b.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[89262],{3905:(e,n,t)=>{t.d(n,{Zo:()=>c,kt:()=>m});var r=t(67294);function a(e,n,t){return n in e?Object.defineProperty(e,n,{value:t,enumerable:!0,configurable:!0,writable:!0}):e[n]=t,e}function o(e,n){var t=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);n&&(r=r.filter((function(n){return Object.getOwnPropertyDescriptor(e,n).enumerable}))),t.push.apply(t,r)}return t}function s(e){for(var n=1;n<arguments.length;n++){var t=null!=arguments[n]?arguments[n]:{};n%2?o(Object(t),!0).forEach((function(n){a(e,n,t[n])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(t)):o(Object(t)).forEach((function(n){Object.defineProperty(e,n,Object.getOwnPropertyDescriptor(t,n))}))}return e}function i(e,n){if(null==e)return{};var t,r,a=function(e,n){if(null==e)return{};var t,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)t=o[r],n.indexOf(t)>=0||(a[t]=e[t]);return a}(e,n);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)t=o[r],n.indexOf(t)>=0||Object.prototype.propertyIsEnumerable.call(e,t)&&(a[t]=e[t])}return a}var l=r.createContext({}),p=function(e){var n=r.useContext(l),t=n;return e&&(t="function"==typeof e?e(n):s(s({},n),e)),t},c=function(e){var n=p(e.components);return r.createElement(l.Provider,{value:n},e.children)},u={inlineCode:"code",wrapper:function(e){var n=e.children;return r.createElement(r.Fragment,{},n)}},d=r.forwardRef((function(e,n){var t=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,c=i(e,["components","mdxType","originalType","parentName"]),d=p(t),m=a,h=d["".concat(l,".").concat(m)]||d[m]||u[m]||o;return t?r.createElement(h,s(s({ref:n},c),{},{components:t})):r.createElement(h,s({ref:n},c))}));function m(e,n){var t=arguments,a=n&&n.mdxType;if("string"==typeof e||a){var o=t.length,s=new Array(o);s[0]=d;var i={};for(var l in n)hasOwnProperty.call(n,l)&&(i[l]=n[l]);i.originalType=e,i.mdxType="string"==typeof e?e:a,s[1]=i;for(var p=2;p<o;p++)s[p]=t[p];return r.createElement.apply(null,s)}return r.createElement.apply(null,t)}d.displayName="MDXCreateElement"},17608:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>s,default:()=>u,frontMatter:()=>o,metadata:()=>i,toc:()=>p});var r=t(83117),a=(t(67294),t(3905));const o={title:"Quickstart - Understand and Search Forms",hide_title:!0,status:"stable"},s="Tutorial: Create a custom search engine and question-answering system",i={unversionedId:"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",id:"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",title:"Quickstart - Understand and Search Forms",description:"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:",source:"@site/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",sourceDirName:"Explore Algorithms/OpenAI",slug:"/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms",draft:!1,tags:[],version:"current",frontMatter:{title:"Quickstart - Understand and Search Forms",hide_title:!0,status:"stable"},sidebar:"docs",previous:{title:"Quickstart - Custom Embeddings and Approximate KNN on GPU",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU"},next:{title:"Getting Started",permalink:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started"}},l={},p=[{value:"1 - Set up dependencies",id:"1---set-up-dependencies",level:2},{value:"2 - Load data into Spark",id:"2---load-data-into-spark",level:2},{value:"3 - Apply form recognition",id:"3---apply-form-recognition",level:2},{value:"4 - Simplify form recognition output",id:"4---simplify-form-recognition-output",level:2},{value:"5 - Add translations",id:"5---add-translations",level:2},{value:"6 - Translate products to emojis with OpenAI \ud83e\udd2f",id:"6---translate-products-to-emojis-with-openai-",level:2},{value:"7 - Infer vendor address continent with OpenAI",id:"7---infer-vendor-address-continent-with-openai",level:2},{value:"8 - Create an Azure Search Index for the Forms",id:"8---create-an-azure-search-index-for-the-forms",level:2},{value:"9 - Try out a search query",id:"9---try-out-a-search-query",level:2},{value:"10 - Build a chatbot that can use Azure Search as a tool \ud83e\udde0\ud83d\udd27",id:"10---build-a-chatbot-that-can-use-azure-search-as-a-tool-",level:2},{value:"11 - Asking our chatbot a question",id:"11---asking-our-chatbot-a-question",level:2},{value:"12 - A quick double check",id:"12---a-quick-double-check",level:2}],c={toc:p};function u(e){let{components:n,...t}=e;return(0,a.kt)("wrapper",(0,r.Z)({},c,t,{components:n,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"tutorial-create-a-custom-search-engine-and-question-answering-system"},"Tutorial: Create a custom search engine and question-answering system"),(0,a.kt)("p",null,"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:"),(0,a.kt)("blockquote",null,(0,a.kt)("ul",{parentName:"blockquote"},(0,a.kt)("li",{parentName:"ul"},"Load various forms (invoices) into a data frame in an Apache Spark session"),(0,a.kt)("li",{parentName:"ul"},"Analyze them to determine their features"),(0,a.kt)("li",{parentName:"ul"},"Assemble the resulting output into a tabular data structure"),(0,a.kt)("li",{parentName:"ul"},"Write the output to a search index hosted in Azure Cognitive Search"),(0,a.kt)("li",{parentName:"ul"},"Explore and query over the content you created"))),(0,a.kt)("h2",{id:"1---set-up-dependencies"},"1 - Set up dependencies"),(0,a.kt)("p",null,"We start by importing packages and connecting to the Azure resources used in this workflow."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"%pip install openai==0.28.1\n")),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.core.platform import find_secret\n\ncognitive_key = find_secret(\n    secret_name="ai-services-api-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string. e.g. cognitive_key="27snaiw..."\ncognitive_location = "eastus"\n\ntranslator_key = find_secret(\n    secret_name="translator-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\ntranslator_location = "eastus"\n\nsearch_key = find_secret(\n    secret_name="azure-search-key", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\nsearch_service = "mmlspark-azure-search"\nsearch_index = "form-demo-index-5"\n\nopenai_key = find_secret(\n    secret_name="openai-api-key-2", keyvault="mmlspark-build-keys"\n)  # Replace the call to find_secret with your key as a python string.\nopenai_service_name = "synapseml-openai-2"\nopenai_deployment_name = "gpt-35-turbo"\nopenai_url = f"https://{openai_service_name}.openai.azure.com/"\n')),(0,a.kt)("h2",{id:"2---load-data-into-spark"},"2 - Load data into Spark"),(0,a.kt)("p",null,"This code loads a few external files from an Azure storage account that's used for demo purposes. The files are various invoices, and they're read into a data frame."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.sql.functions import udf\nfrom pyspark.sql.types import StringType\n\n\ndef blob_to_url(blob):\n    [prefix, postfix] = blob.split("@")\n    container = prefix.split("/")[-1]\n    split_postfix = postfix.split("/")\n    account = split_postfix[0]\n    filepath = "/".join(split_postfix[1:])\n    return "https://{}/{}/{}".format(account, container, filepath)\n\n\ndf2 = (\n    spark.read.format("binaryFile")\n    .load("wasbs://publicwasb@mmlspark.blob.core.windows.net/form_subset/*")\n    .select("path")\n    .limit(10)\n    .select(udf(blob_to_url, StringType())("path").alias("url"))\n    .cache()\n)\n\ndisplay(df2)\n')),(0,a.kt)("img",{src:"https://mmlspark.blob.core.windows.net/graphics/Invoice11205.svg",width:"40%"}),(0,a.kt)("h2",{id:"3---apply-form-recognition"},"3 - Apply form recognition"),(0,a.kt)("p",null,"This code loads the ",(0,a.kt)("a",{parentName:"p",href:"https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#form-recognizer"},"AnalyzeInvoices transformer")," and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.form import AnalyzeInvoices\n\nanalyzed_df = (\n    AnalyzeInvoices()\n    .setSubscriptionKey(cognitive_key)\n    .setLocation(cognitive_location)\n    .setImageUrlCol("url")\n    .setOutputCol("invoices")\n    .setErrorCol("errors")\n    .setConcurrency(5)\n    .transform(df2)\n    .cache()\n)\n\ndisplay(analyzed_df)\n')),(0,a.kt)("h2",{id:"4---simplify-form-recognition-output"},"4 - Simplify form recognition output"),(0,a.kt)("p",null,"This code uses the ",(0,a.kt)("a",{parentName:"p",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.FormOntologyTransformer"},"FormOntologyLearner"),", a transformer that analyzes the output of Form Recognizer transformers (for Azure AI Document Intelligence) and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content."),(0,a.kt)("p",null,"FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.form import FormOntologyLearner\n\norganized_df = (\n    FormOntologyLearner()\n    .setInputCol("invoices")\n    .setOutputCol("extracted")\n    .fit(analyzed_df)\n    .transform(analyzed_df)\n    .select("url", "extracted.*")\n    .cache()\n)\n\ndisplay(organized_df)\n')),(0,a.kt)("p",null,"With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.sql.functions import explode, col\n\nitemized_df = (\n    organized_df.select("*", explode(col("Items")).alias("Item"))\n    .drop("Items")\n    .select("Item.*", "*")\n    .drop("Item")\n)\n\ndisplay(itemized_df)\n')),(0,a.kt)("h2",{id:"5---add-translations"},"5 - Add translations"),(0,a.kt)("p",null,"This code loads ",(0,a.kt)("a",{parentName:"p",href:"https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#translation"},"Translate"),', a transformer that calls the Azure AI Translator service in Azure AI services. The original text, which is in English in the "Description" column, is machine-translated into various languages. All of the output is consolidated into "output.translations" array.'),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.translate import Translate\n\ntranslated_df = (\n    Translate()\n    .setSubscriptionKey(translator_key)\n    .setLocation(translator_location)\n    .setTextCol("Description")\n    .setErrorCol("TranslationError")\n    .setOutputCol("output")\n    .setToLanguage(["zh-Hans", "fr", "ru", "cy"])\n    .setConcurrency(5)\n    .transform(itemized_df)\n    .withColumn("Translations", col("output.translations")[0])\n    .drop("output", "TranslationError")\n    .cache()\n)\n\ndisplay(translated_df)\n')),(0,a.kt)("h2",{id:"6---translate-products-to-emojis-with-openai-"},"6 - Translate products to emojis with OpenAI \ud83e\udd2f"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.openai import OpenAIPrompt\nfrom pyspark.sql.functions import trim, split\n\nemoji_template = """ \n  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma\n  \n  Two Ducks: \ud83e\udd86\ud83e\udd86,\n  Light Bulb: \ud83d\udca1,\n  Three Peaches: \ud83c\udf51\ud83c\udf51\ud83c\udf51,\n  Two kitchen stoves: \u2668\ufe0f\u2668\ufe0f,\n  A red car: \ud83d\ude97,\n  A person and a cat: \ud83e\uddcd\ud83d\udc08,\n  A {Description}: """\n\nprompter = (\n    OpenAIPrompt()\n    .setSubscriptionKey(openai_key)\n    .setDeploymentName(openai_deployment_name)\n    .setUrl(openai_url)\n    .setMaxTokens(5)\n    .setPromptTemplate(emoji_template)\n    .setErrorCol("error")\n    .setOutputCol("Emoji")\n)\n\nemoji_df = (\n    prompter.transform(translated_df)\n    .withColumn("Emoji", trim(split(col("Emoji"), ",").getItem(0)))\n    .drop("error", "prompt")\n    .cache()\n)\n')),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(emoji_df.select("Description", "Emoji"))\n')),(0,a.kt)("h2",{id:"7---infer-vendor-address-continent-with-openai"},"7 - Infer vendor address continent with OpenAI"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'continent_template = """\nWhich continent does the following address belong to? \n\nPick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. \n\nDont respond with anything but one of the above. If you don\'t know the answer or cannot figure it out from the text, return None. End your answer with a comma.\n\nAddress: "6693 Ryan Rd, North Whales",\nContinent: Europe,\nAddress: "6693 Ryan Rd",\nContinent: None,\nAddress: "{VendorAddress}",\nContinent:"""\n\ncontinent_df = (\n    prompter.setOutputCol("Continent")\n    .setPromptTemplate(continent_template)\n    .transform(emoji_df)\n    .withColumn("Continent", trim(split(col("Continent"), ",").getItem(0)))\n    .drop("error", "prompt")\n    .cache()\n)\n')),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(continent_df.select("VendorAddress", "Continent"))\n')),(0,a.kt)("h2",{id:"8---create-an-azure-search-index-for-the-forms"},"8 - Create an Azure Search Index for the Forms"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services import *\nfrom pyspark.sql.functions import monotonically_increasing_id, lit\n\n(\n    continent_df.withColumn("DocID", monotonically_increasing_id().cast("string"))\n    .withColumn("SearchAction", lit("upload"))\n    .writeToAzureSearch(\n        subscriptionKey=search_key,\n        actionCol="SearchAction",\n        serviceName=search_service,\n        indexName=search_index,\n        keyCol="DocID",\n    )\n)\n')),(0,a.kt)("h2",{id:"9---try-out-a-search-query"},"9 - Try out a search query"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import requests\n\nsearch_url = "https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06".format(\n    search_service, search_index\n)\nrequests.post(\n    search_url, json={"search": "door"}, headers={"api-key": search_key}\n).json()\n')),(0,a.kt)("h2",{id:"10---build-a-chatbot-that-can-use-azure-search-as-a-tool-"},"10 - Build a chatbot that can use Azure Search as a tool \ud83e\udde0\ud83d\udd27"),(0,a.kt)("img",{src:"https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg",width:"40%"}),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import json\nimport openai\n\nopenai.api_type = "azure"\nopenai.api_base = openai_url\nopenai.api_key = openai_key\nopenai.api_version = "2023-03-15-preview"\n\nchat_context_prompt = f"""\nYou are a chatbot designed to answer questions with the help of a search engine that has the following information:\n\n{continent_df.columns}\n\nIf you dont know the answer to a question say "I dont know". Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{"query": "example_query"}}\n"""\n\n\ndef search_query_prompt(question):\n    return f"""\nGiven the search engine above, what would you search for to answer the following question?\n\nQuestion: "{question}"\n\nPlease output a json in the form of {{"query": "example_query"}}\n"""\n\n\ndef search_result_prompt(query):\n    search_results = requests.post(\n        search_url, json={"search": query}, headers={"api-key": search_key}\n    ).json()\n    return f"""\n\nYou previously ran a search for "{query}" which returned the following results:\n\n{search_results}\n\nYou should use the results to help you answer questions. If you dont know the answer to a question say "I dont know". Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. \n"""\n\n\ndef prompt_gpt(messages):\n    response = openai.ChatCompletion.create(\n        engine=openai_deployment_name, messages=messages, max_tokens=None, top_p=0.95\n    )\n    return response["choices"][0]["message"]["content"]\n\n\ndef custom_chatbot(question):\n    while True:\n        try:\n            query = json.loads(\n                prompt_gpt(\n                    [\n                        {"role": "system", "content": chat_context_prompt},\n                        {"role": "user", "content": search_query_prompt(question)},\n                    ]\n                )\n            )["query"]\n\n            return prompt_gpt(\n                [\n                    {"role": "system", "content": chat_context_prompt},\n                    {"role": "system", "content": search_result_prompt(query)},\n                    {"role": "user", "content": question},\n                ]\n            )\n        except Exception as e:\n            raise e\n')),(0,a.kt)("h2",{id:"11---asking-our-chatbot-a-question"},"11 - Asking our chatbot a question"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'custom_chatbot("What did Luke Diaz buy?")\n')),(0,a.kt)("h2",{id:"12---a-quick-double-check"},"12 - A quick double check"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'display(\n    continent_df.where(col("CustomerName") == "Luke Diaz")\n    .select("Description")\n    .distinct()\n)\n')))}u.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/935f2afb.0c4b2f48.js b/assets/js/935f2afb.0c4b2f48.js
new file mode 100644
index 0000000000..d9d45fc11b
--- /dev/null
+++ b/assets/js/935f2afb.0c4b2f48.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[80053],{1109:e=>{e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"Next","banner":"unreleased","badge":true,"noIndex":false,"className":"docs-version-current","isLast":false,"docsSidebars":{"docs":[{"type":"link","label":"What is SynapseML?","href":"/SynapseML/docs/next/Overview","docId":"Overview"},{"type":"category","label":"Get Started","items":[{"type":"link","label":"Create a Spark Cluster","href":"/SynapseML/docs/next/Get Started/Create a Spark Cluster","docId":"Get Started/Create a Spark Cluster"},{"type":"link","label":"Install SynapseML","href":"/SynapseML/docs/next/Get Started/Install SynapseML","docId":"Get Started/Install SynapseML"},{"type":"link","label":"Set up Cognitive Services","href":"/SynapseML/docs/next/Get Started/Set up Cognitive Services","docId":"Get Started/Set up Cognitive Services"},{"type":"link","label":"Quickstart - Your First Models","href":"/SynapseML/docs/next/Get Started/Quickstart - Your First Models","docId":"Get Started/Quickstart - Your First Models"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Explore Algorithms","items":[{"type":"category","label":"LightGBM","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview","docId":"Explore Algorithms/LightGBM/Overview"},{"type":"link","label":"Quickstart - Classification, Ranking, and Regression","href":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","docId":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression"}],"collapsed":true,"collapsible":true},{"type":"category","label":"AI Services","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview","docId":"Explore Algorithms/AI Services/Overview"},{"type":"link","label":"Geospatial Services","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services","docId":"Explore Algorithms/AI Services/Geospatial Services"},{"type":"link","label":"Multivariate Anomaly Detection","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection","docId":"Explore Algorithms/AI Services/Multivariate Anomaly Detection"},{"type":"link","label":"Advanced Usage - Async, Batching, and Multi-Key","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","docId":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key"},{"type":"link","label":"Quickstart - Analyze Celebrity Quotes","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","docId":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes"},{"type":"link","label":"Quickstart - Analyze Text","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text","docId":"Explore Algorithms/AI Services/Quickstart - Analyze Text"},{"type":"link","label":"Quickstart - Create a Visual Search Engine","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","docId":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine"},{"type":"link","label":"Quickstart - Create Audiobooks","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","docId":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks"},{"type":"link","label":"Quickstart - Document Question and Answering with PDFs","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","docId":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs"},{"type":"link","label":"Quickstart - Flooding Risk","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk","docId":"Explore Algorithms/AI Services/Quickstart - Flooding Risk"},{"type":"link","label":"Quickstart - Predictive Maintenance","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","docId":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance"}],"collapsed":true,"collapsible":true},{"type":"category","label":"OpenAI","items":[{"type":"link","label":"Langchain","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain","docId":"Explore Algorithms/OpenAI/Langchain"},{"type":"link","label":"OpenAI","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/","docId":"Explore Algorithms/OpenAI/OpenAI"},{"type":"link","label":"Quickstart - OpenAI Embedding","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","docId":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding"},{"type":"link","label":"Quickstart - OpenAI Embedding and GPU based KNN","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","docId":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN"},{"type":"link","label":"Quickstart - Custom Embeddings and Approximate KNN on GPU","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU","docId":"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU"},{"type":"link","label":"Quickstart - Understand and Search Forms","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","docId":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Deep Learning","items":[{"type":"link","label":"Getting Started","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started","docId":"Explore Algorithms/Deep Learning/Getting Started"},{"type":"link","label":"ONNX","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX","docId":"Explore Algorithms/Deep Learning/ONNX"},{"type":"link","label":"Distributed Training","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training","docId":"Explore Algorithms/Deep Learning/Distributed Training"},{"type":"link","label":"Quickstart - Fine-tune a Text Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","docId":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier"},{"type":"link","label":"Quickstart - Fine-tune a Vision Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","docId":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier"},{"type":"link","label":"Quickstart - ONNX Model Inference","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","docId":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference"},{"type":"link","label":"Quickstart - Transfer Learn for Image Classification","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","docId":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Responsible AI","items":[{"type":"link","label":"Interpreting Model Predictions","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions","docId":"Explore Algorithms/Responsible AI/Interpreting Model Predictions"},{"type":"link","label":"Tabular Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers","docId":"Explore Algorithms/Responsible AI/Tabular Explainers"},{"type":"link","label":"Text Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers","docId":"Explore Algorithms/Responsible AI/Text Explainers"},{"type":"link","label":"Image Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers","docId":"Explore Algorithms/Responsible AI/Image Explainers"},{"type":"link","label":"PDP and ICE Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers","docId":"Explore Algorithms/Responsible AI/PDP and ICE Explainers"},{"type":"link","label":"Data Balance Analysis","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis","docId":"Explore Algorithms/Responsible AI/Data Balance Analysis"},{"type":"link","label":"Explanation Dashboard","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard","docId":"Explore Algorithms/Responsible AI/Explanation Dashboard"},{"type":"link","label":"Quickstart - Data Balance Analysis","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","docId":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis"},{"type":"link","label":"Quickstart - Snow Leopard Detection","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","docId":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Causal Inference","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview","docId":"Explore Algorithms/Causal Inference/Overview"},{"type":"link","label":"Quickstart - Measure Causal Effects","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","docId":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects"},{"type":"link","label":"Quickstart - Measure Heterogeneous Effects","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","docId":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects"},{"type":"link","label":"Quickstart - Synthetic difference in differences","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","docId":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Classification","items":[{"type":"link","label":"Quickstart - Train Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier","docId":"Explore Algorithms/Classification/Quickstart - Train Classifier"},{"type":"link","label":"Quickstart - SparkML vs SynapseML","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","docId":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML"},{"type":"link","label":"Quickstart - Vowpal Wabbit on Tabular Data","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","docId":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data"},{"type":"link","label":"Quickstart - Vowpal Wabbit on Text Data","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","docId":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Regression","items":[{"type":"link","label":"Quickstart - Data Cleaning","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning","docId":"Explore Algorithms/Regression/Quickstart - Data Cleaning"},{"type":"link","label":"Quickstart - Train Regressor","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor","docId":"Explore Algorithms/Regression/Quickstart - Train Regressor"},{"type":"link","label":"Quickstart - Vowpal Wabbit and LightGBM","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","docId":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Anomaly Detection","items":[{"type":"link","label":"Quickstart - Isolation Forests","href":"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","docId":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Hyperparameter Tuning","items":[{"type":"link","label":"HyperOpt","href":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt","docId":"Explore Algorithms/Hyperparameter Tuning/HyperOpt"},{"type":"link","label":"Quickstart - Random Search","href":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","docId":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search"}],"collapsed":true,"collapsible":true},{"type":"category","label":"OpenCV","items":[{"type":"link","label":"Image Transformations","href":"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations","docId":"Explore Algorithms/OpenCV/Image Transformations"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Vowpal Wabbit","items":[{"type":"link","label":"About","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview","docId":"Explore Algorithms/Vowpal Wabbit/Overview"},{"type":"link","label":"Multi-class classification","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification","docId":"Explore Algorithms/Vowpal Wabbit/Multi-class classification"},{"type":"link","label":"Contextual Bandits","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","docId":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits"},{"type":"link","label":"Quickstart - Classification, Quantile Regression, and Regression","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression"},{"type":"link","label":"Quickstart - Classification using SparkML Vectors","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors"},{"type":"link","label":"Quickstart - Classification using VW-native Format","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Other Algorithms","items":[{"type":"link","label":"SAR Algorithm","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","docId":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations"},{"type":"link","label":"CyberML","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML","docId":"Explore Algorithms/Other Algorithms/Cyber ML"},{"type":"link","label":"Quickstart - Anomalous Access Detection","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","docId":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection"},{"type":"link","label":"Quickstart - Exploring Art Across Cultures","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","docId":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures"}],"collapsed":true,"collapsible":true}],"collapsed":true,"collapsible":true},{"type":"category","label":"Use with MLFlow","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Use with MLFlow/Overview","docId":"Use with MLFlow/Overview"},{"type":"link","label":"Install","href":"/SynapseML/docs/next/Use with MLFlow/Install","docId":"Use with MLFlow/Install"},{"type":"link","label":"Autologging","href":"/SynapseML/docs/next/Use with MLFlow/Autologging","docId":"Use with MLFlow/Autologging"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Deploy Models","items":[{"type":"link","label":"About","href":"/SynapseML/docs/next/Deploy Models/Overview","docId":"Deploy Models/Overview"},{"type":"link","label":"Quickstart - Deploying a Classifier","href":"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier","docId":"Deploy Models/Quickstart - Deploying a Classifier"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Reference","items":[{"type":"link","label":"Contributor Guide","href":"/SynapseML/docs/next/Reference/Contributor Guide","docId":"Reference/Contributor Guide"},{"type":"link","label":"Developer Setup","href":"/SynapseML/docs/next/Reference/Developer Setup","docId":"Reference/Developer Setup"},{"type":"link","label":"Docker Setup","href":"/SynapseML/docs/next/Reference/Docker Setup","docId":"Reference/Docker Setup"},{"type":"link","label":"R setup","href":"/SynapseML/docs/next/Reference/R Setup","docId":"Reference/R Setup"}],"collapsed":true,"collapsible":true}]},"docs":{"Deploy Models/Overview":{"id":"Deploy Models/Overview","title":"Spark Serving","description":"An Engine for Deploying Spark Jobs as Distributed Web Services","sidebar":"docs"},"Deploy Models/Quickstart - Deploying a Classifier":{"id":"Deploy Models/Quickstart - Deploying a Classifier","title":"Quickstart - Deploying a Classifier","description":"Model Deployment with Spark Serving","sidebar":"docs"},"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key":{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","title":"Advanced Usage - Async, Batching, and Multi-Key","description":"Step 1: Imports and Keys","sidebar":"docs"},"Explore Algorithms/AI Services/Geospatial Services":{"id":"Explore Algorithms/AI Services/Geospatial Services","title":"Geospatial Services","description":"Microsoft Azure Maps  provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.","sidebar":"docs"},"Explore Algorithms/AI Services/Multivariate Anomaly Detection":{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","title":"Multivariate Anomaly Detection","description":"This recipe shows how you can use SynapseML and Azure AI services on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI services, and we then use to the model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.","sidebar":"docs"},"Explore Algorithms/AI Services/Overview":{"id":"Explore Algorithms/AI Services/Overview","title":"Overview","description":"Azure AI services help developers and organizations rapidly create intelligent, cutting-edge, market-ready, and responsible applications with out-of-the-box and pre-built and customizable APIs and models.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes":{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","title":"Quickstart - Analyze Celebrity Quotes","description":"Extracting celebrity quote images using Bing Image Search on Spark","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Analyze Text":{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","title":"Quickstart - Analyze Text","description":"Azure AI Language is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine":{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","title":"Quickstart - Create a Visual Search Engine","description":"Creating a searchable Art Database with The MET\'s open-access collection","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Create Audiobooks":{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","title":"Quickstart - Create Audiobooks","description":"Step 1: Load libraries and add service information","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs":{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","title":"Quickstart - Document Question and Answering with PDFs","description":"Introduction","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Flooding Risk":{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","title":"Quickstart - Flooding Risk","description":"King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance":{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","title":"Quickstart - Predictive Maintenance","description":"This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We\'ll follow along with the CosmosDB and Synapse Link sample. To keep things simple, in this recipe we\'ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.","sidebar":"docs"},"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests":{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","title":"Quickstart - Isolation Forests","description":"This article shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Overview":{"id":"Explore Algorithms/Causal Inference/Overview","title":"Overview","description":"Causal Inference on Apache Spark","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects":{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","title":"Quickstart - Measure Causal Effects","description":"This sample notebook aims to show the application of using SynapseML\'s DoubleMLEstimator for inferring causality using observational data.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects":{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","title":"Quickstart - Measure Heterogeneous Effects","description":"This sample notebook aims to show the application of using SynapseML\'s DoubleMLEstimator for inferring causality using observational data.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences":{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","title":"Quickstart - Synthetic difference in differences","description":"This sample notebook aims to show readers how to use SynapseML\'s DiffInDiffEstimator, SyntheticControlEstimator and SyntheticDiffInDiffEstimator to estimate the causal effect of a treatment on a particular outcome.","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML":{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","title":"Quickstart - SparkML vs SynapseML","description":"In this article, you perform the same classification task in two","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Train Classifier":{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","title":"Quickstart - Train Classifier","description":"Classification - Adult Census","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data":{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","title":"Quickstart - Vowpal Wabbit on Tabular Data","description":"In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) classifier in SynapseML.","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data":{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","title":"Quickstart - Vowpal Wabbit on Text Data","description":"In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is Sentiment140 twitter data. First, we import a few packages that we need.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Distributed Training":{"id":"Explore Algorithms/Deep Learning/Distributed Training","title":"Distributed Training","description":"Why Simple Deep Learning","sidebar":"docs"},"Explore Algorithms/Deep Learning/Getting Started":{"id":"Explore Algorithms/Deep Learning/Getting Started","title":"Getting Started","description":"This is a sample with databricks 10.4.x-gpu-ml-scala2.12 runtime","sidebar":"docs"},"Explore Algorithms/Deep Learning/ONNX":{"id":"Explore Algorithms/Deep Learning/ONNX","title":"ONNX","description":"Learn how to use the ONNX model transformer to run inference for an ONNX model on Spark.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier":{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","title":"Quickstart - Fine-tune a Text Classifier","description":"Deep Learning - Deep Text Classifier","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier":{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","title":"Quickstart - Fine-tune a Vision Classifier","description":"Deep Learning - Deep Vision Classifier","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference":{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","title":"Quickstart - ONNX Model Inference","description":"In this example, you train a LightGBM model and convert the model to ONNX format. Once converted, you use the model to infer some testing data on Spark.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification":{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","title":"Quickstart - Transfer Learn for Image Classification","description":"Deep Learning - Flower Image Classification","sidebar":"docs"},"Explore Algorithms/Hyperparameter Tuning/HyperOpt":{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","title":"HyperOpt","description":"SynapseML is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others.","sidebar":"docs"},"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search":{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","title":"Quickstart - Random Search","description":"This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we\'ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer.","sidebar":"docs"},"Explore Algorithms/LightGBM/Overview":{"id":"Explore Algorithms/LightGBM/Overview","title":"Overview","description":"LightGBM","sidebar":"docs"},"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression":{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","title":"Quickstart - Classification, Ranking, and Regression","description":"What is LightGBM","sidebar":"docs"},"Explore Algorithms/OpenAI/Langchain":{"id":"Explore Algorithms/OpenAI/Langchain","title":"Langchain","description":"LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.","sidebar":"docs"},"Explore Algorithms/OpenAI/OpenAI":{"id":"Explore Algorithms/OpenAI/OpenAI","title":"OpenAI","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure OpenAI.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU":{"id":"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU","title":"Quickstart - Custom Embeddings and Approximate KNN on GPU","description":"The demo extending existing Azure OpenAI based demo when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using multilingual E5 text embeddings and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using NVIDIA TensorRT and Spark Rapids ML. The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to previos CPU based demo","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding":{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","title":"Quickstart - OpenAI Embedding","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN":{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","title":"Quickstart - OpenAI Embedding and GPU based KNN","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library Spark Rapids ML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on \\"Quickstart - OpenAI Embedding\\" notebook with NVIDIA GPU accelerated KNN.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms":{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","title":"Quickstart - Understand and Search Forms","description":"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:","sidebar":"docs"},"Explore Algorithms/OpenCV/Image Transformations":{"id":"Explore Algorithms/OpenCV/Image Transformations","title":"Image Transformations","description":"OpenCV - Pipeline Image Transformations","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Cyber ML":{"id":"Explore Algorithms/Other Algorithms/Cyber ML","title":"CyberML","description":"access anomalies: complementaccess.py","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection":{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","title":"Quickstart - Anomalous Access Detection","description":"Here we demonstrate a novel CyberML model which can learn user access patterns and then automatically detect anomalous user access based on learned behavior.","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures":{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","title":"Quickstart - Exploring Art Across Cultures","description":"This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations":{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","title":"Smart Adaptive Recommendations (SAR) Algorithm","description":"The following document is a subset of the implemented logic. The original can be found here","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Data Cleaning":{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","title":"Quickstart - Data Cleaning","description":"Regression -  Flight Delays with DataCleaning","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Train Regressor":{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","title":"Quickstart - Train Regressor","description":"Regression - Auto Imports","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM":{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","title":"Quickstart - Vowpal Wabbit and LightGBM","description":"This notebook shows how to build simple regression models by using","sidebar":"docs"},"Explore Algorithms/Responsible AI/Data Balance Analysis":{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","title":"Data Balance Analysis on Spark","description":"Learn how to do Data Balance Analysis on Spark to determine how well features and feature values are represented in your dataset.","sidebar":"docs"},"Explore Algorithms/Responsible AI/Explanation Dashboard":{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","title":"Explanation Dashboard","description":"Interpretability - Explanation Dashboard","sidebar":"docs"},"Explore Algorithms/Responsible AI/Image Explainers":{"id":"Explore Algorithms/Responsible AI/Image Explainers","title":"Image Explainers","description":"Interpretability - Image Explainers","sidebar":"docs"},"Explore Algorithms/Responsible AI/Interpreting Model Predictions":{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","title":"Interpreting Model Predictions","description":"Interpretable Machine Learning","sidebar":"docs"},"Explore Algorithms/Responsible AI/PDP and ICE Explainers":{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","title":"PDP and ICE Explainers","description":"Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots","sidebar":"docs"},"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis":{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","title":"Quickstart - Data Balance Analysis","description":"Data Balance Analysis using the Adult Census Income dataset","sidebar":"docs"},"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection":{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","title":"Quickstart - Snow Leopard Detection","description":"Automated Snow Leopard Detection with Synapse Machine Learning","sidebar":"docs"},"Explore Algorithms/Responsible AI/Tabular Explainers":{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","title":"Tabular Explainers","description":"In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.","sidebar":"docs"},"Explore Algorithms/Responsible AI/Text Explainers":{"id":"Explore Algorithms/Responsible AI/Text Explainers","title":"Text Explainers","description":"Interpretability - Text Explainers","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Contextual Bandits":{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","title":"Contextual Bandits","description":"In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Multi-class classification":{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","title":"Multi-class classification","description":"Read dataset","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Overview":{"id":"Explore Algorithms/Vowpal Wabbit/Overview","title":"VW","description":"Overview","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","title":"Quickstart - Classification using SparkML Vectors","description":"SparkML Vector input","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","title":"Quickstart - Classification using VW-native Format","description":"SparkML Vector input","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","title":"Quickstart - Classification, Quantile Regression, and Regression","description":"VowpalWabbit (VW) is a machine learning system which","sidebar":"docs"},"Get Started/Create a Spark Cluster":{"id":"Get Started/Create a Spark Cluster","title":"Create a Spark Cluster","description":"SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.","sidebar":"docs"},"Get Started/Install SynapseML":{"id":"Get Started/Install SynapseML","title":"Install SynapseML","description":"Install SynapseML","sidebar":"docs"},"Get Started/Quickstart - Your First Models":{"id":"Get Started/Quickstart - Your First Models","title":"Quickstart - Your First Models","description":"This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.","sidebar":"docs"},"Get Started/Set up Cognitive Services":{"id":"Get Started/Set up Cognitive Services","title":"Set up Cognitive Services","description":"In order to use SynapseML\'s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.","sidebar":"docs"},"Overview":{"id":"Overview","title":"What is SynapseML?","description":"SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the Apache Spark distributed computing framework and shares the same API as the SparkML/MLLib library, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.","sidebar":"docs"},"Quick Examples/estimators/estimators_causal":{"id":"Quick Examples/estimators/estimators_causal","title":"Estimators - Causal","description":""},"Quick Examples/estimators/estimators_cognitive":{"id":"Quick Examples/estimators/estimators_cognitive","title":"Estimators - Cognitive","description":""},"Quick Examples/estimators/estimators_core":{"id":"Quick Examples/estimators/estimators_core","title":"Estimators - Core","description":""},"Quick Examples/estimators/estimators_lightgbm":{"id":"Quick Examples/estimators/estimators_lightgbm","title":"Estimators - LightGBM","description":""},"Quick Examples/estimators/estimators_vw":{"id":"Quick Examples/estimators/estimators_vw","title":"Estimators - Vowpal Wabbit","description":""},"Quick Examples/transformers/transformers_cognitive":{"id":"Quick Examples/transformers/transformers_cognitive","title":"Transformers - Cognitive","description":""},"Quick Examples/transformers/transformers_core":{"id":"Quick Examples/transformers/transformers_core","title":"Transformers - Core","description":""},"Quick Examples/transformers/transformers_deep_learning":{"id":"Quick Examples/transformers/transformers_deep_learning","title":"Deep Learning","description":""},"Quick Examples/transformers/transformers_opencv":{"id":"Quick Examples/transformers/transformers_opencv","title":"Transformers - OpenCV","description":""},"Quick Examples/transformers/transformers_vw":{"id":"Quick Examples/transformers/transformers_vw","title":"Transformers - Vowpal Wabbit","description":""},"Reference/Contributor Guide":{"id":"Reference/Contributor Guide","title":"Contributor Guide","description":"Contributor Guide","sidebar":"docs"},"Reference/Developer Setup":{"id":"Reference/Developer Setup","title":"Developer Setup","description":"Developer Setup","sidebar":"docs"},"Reference/Docker Setup":{"id":"Reference/Docker Setup","title":"Docker Setup","description":"Docker Setup","sidebar":"docs"},"Reference/Dotnet Setup":{"id":"Reference/Dotnet Setup","title":".NET setup","description":".NET setup"},"Reference/Quickstart - LightGBM in Dotnet":{"id":"Reference/Quickstart - LightGBM in Dotnet","title":"Quickstart - LightGBM in Dotnet","description":"A simple example about classification with LightGBMClassifier using .NET"},"Reference/R Setup":{"id":"Reference/R Setup","title":"R setup","description":"R setup and example for SynapseML","sidebar":"docs"},"Use with MLFlow/Autologging":{"id":"Use with MLFlow/Autologging","title":"Autologging","description":"SynapseML autologging","sidebar":"docs"},"Use with MLFlow/Install":{"id":"Use with MLFlow/Install","title":"Install","description":"install Mlflow on different environments","sidebar":"docs"},"Use with MLFlow/Overview":{"id":"Use with MLFlow/Overview","title":"Overview","description":"MLflow support of SynapseML","sidebar":"docs"}}}')}}]);
\ No newline at end of file
diff --git a/assets/js/935f2afb.57f575fd.js b/assets/js/935f2afb.57f575fd.js
deleted file mode 100644
index f647654866..0000000000
--- a/assets/js/935f2afb.57f575fd.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[80053],{1109:e=>{e.exports=JSON.parse('{"pluginId":"default","version":"current","label":"Next","banner":"unreleased","badge":true,"noIndex":false,"className":"docs-version-current","isLast":false,"docsSidebars":{"docs":[{"type":"link","label":"What is SynapseML?","href":"/SynapseML/docs/next/Overview","docId":"Overview"},{"type":"category","label":"Get Started","items":[{"type":"link","label":"Create a Spark Cluster","href":"/SynapseML/docs/next/Get Started/Create a Spark Cluster","docId":"Get Started/Create a Spark Cluster"},{"type":"link","label":"Install SynapseML","href":"/SynapseML/docs/next/Get Started/Install SynapseML","docId":"Get Started/Install SynapseML"},{"type":"link","label":"Set up Cognitive Services","href":"/SynapseML/docs/next/Get Started/Set up Cognitive Services","docId":"Get Started/Set up Cognitive Services"},{"type":"link","label":"Quickstart - Your First Models","href":"/SynapseML/docs/next/Get Started/Quickstart - Your First Models","docId":"Get Started/Quickstart - Your First Models"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Explore Algorithms","items":[{"type":"category","label":"LightGBM","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview","docId":"Explore Algorithms/LightGBM/Overview"},{"type":"link","label":"Quickstart - Classification, Ranking, and Regression","href":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","docId":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression"}],"collapsed":true,"collapsible":true},{"type":"category","label":"AI Services","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview","docId":"Explore Algorithms/AI Services/Overview"},{"type":"link","label":"Geospatial Services","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services","docId":"Explore Algorithms/AI Services/Geospatial Services"},{"type":"link","label":"Multivariate Anomaly Detection","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection","docId":"Explore Algorithms/AI Services/Multivariate Anomaly Detection"},{"type":"link","label":"Advanced Usage - Async, Batching, and Multi-Key","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","docId":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key"},{"type":"link","label":"Quickstart - Analyze Celebrity Quotes","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","docId":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes"},{"type":"link","label":"Quickstart - Analyze Text","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text","docId":"Explore Algorithms/AI Services/Quickstart - Analyze Text"},{"type":"link","label":"Quickstart - Create a Visual Search Engine","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","docId":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine"},{"type":"link","label":"Quickstart - Create Audiobooks","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","docId":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks"},{"type":"link","label":"Quickstart - Document Question and Answering with PDFs","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","docId":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs"},{"type":"link","label":"Quickstart - Flooding Risk","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk","docId":"Explore Algorithms/AI Services/Quickstart - Flooding Risk"},{"type":"link","label":"Quickstart - Predictive Maintenance","href":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","docId":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance"}],"collapsed":true,"collapsible":true},{"type":"category","label":"OpenAI","items":[{"type":"link","label":"Langchain","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain","docId":"Explore Algorithms/OpenAI/Langchain"},{"type":"link","label":"OpenAI","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/","docId":"Explore Algorithms/OpenAI/OpenAI"},{"type":"link","label":"Quickstart - OpenAI Embedding","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","docId":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding"},{"type":"link","label":"Quickstart - OpenAI Embedding and GPU based KNN","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","docId":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN"},{"type":"link","label":"Quickstart - Understand and Search Forms","href":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","docId":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Deep Learning","items":[{"type":"link","label":"Getting Started","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started","docId":"Explore Algorithms/Deep Learning/Getting Started"},{"type":"link","label":"ONNX","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX","docId":"Explore Algorithms/Deep Learning/ONNX"},{"type":"link","label":"Distributed Training","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training","docId":"Explore Algorithms/Deep Learning/Distributed Training"},{"type":"link","label":"Quickstart - Fine-tune a Text Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","docId":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier"},{"type":"link","label":"Quickstart - Fine-tune a Vision Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","docId":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier"},{"type":"link","label":"Quickstart - ONNX Model Inference","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","docId":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference"},{"type":"link","label":"Quickstart - Transfer Learn for Image Classification","href":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","docId":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Responsible AI","items":[{"type":"link","label":"Interpreting Model Predictions","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions","docId":"Explore Algorithms/Responsible AI/Interpreting Model Predictions"},{"type":"link","label":"Tabular Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers","docId":"Explore Algorithms/Responsible AI/Tabular Explainers"},{"type":"link","label":"Text Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers","docId":"Explore Algorithms/Responsible AI/Text Explainers"},{"type":"link","label":"Image Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers","docId":"Explore Algorithms/Responsible AI/Image Explainers"},{"type":"link","label":"PDP and ICE Explainers","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers","docId":"Explore Algorithms/Responsible AI/PDP and ICE Explainers"},{"type":"link","label":"Data Balance Analysis","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis","docId":"Explore Algorithms/Responsible AI/Data Balance Analysis"},{"type":"link","label":"Explanation Dashboard","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard","docId":"Explore Algorithms/Responsible AI/Explanation Dashboard"},{"type":"link","label":"Quickstart - Data Balance Analysis","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","docId":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis"},{"type":"link","label":"Quickstart - Snow Leopard Detection","href":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","docId":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Causal Inference","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview","docId":"Explore Algorithms/Causal Inference/Overview"},{"type":"link","label":"Quickstart - Measure Causal Effects","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","docId":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects"},{"type":"link","label":"Quickstart - Measure Heterogeneous Effects","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","docId":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects"},{"type":"link","label":"Quickstart - Synthetic difference in differences","href":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","docId":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Classification","items":[{"type":"link","label":"Quickstart - Train Classifier","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier","docId":"Explore Algorithms/Classification/Quickstart - Train Classifier"},{"type":"link","label":"Quickstart - SparkML vs SynapseML","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","docId":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML"},{"type":"link","label":"Quickstart - Vowpal Wabbit on Tabular Data","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","docId":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data"},{"type":"link","label":"Quickstart - Vowpal Wabbit on Text Data","href":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","docId":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Regression","items":[{"type":"link","label":"Quickstart - Data Cleaning","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning","docId":"Explore Algorithms/Regression/Quickstart - Data Cleaning"},{"type":"link","label":"Quickstart - Train Regressor","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor","docId":"Explore Algorithms/Regression/Quickstart - Train Regressor"},{"type":"link","label":"Quickstart - Vowpal Wabbit and LightGBM","href":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","docId":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Anomaly Detection","items":[{"type":"link","label":"Quickstart - Isolation Forests","href":"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","docId":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Hyperparameter Tuning","items":[{"type":"link","label":"HyperOpt","href":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt","docId":"Explore Algorithms/Hyperparameter Tuning/HyperOpt"},{"type":"link","label":"Quickstart - Random Search","href":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","docId":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search"}],"collapsed":true,"collapsible":true},{"type":"category","label":"OpenCV","items":[{"type":"link","label":"Image Transformations","href":"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations","docId":"Explore Algorithms/OpenCV/Image Transformations"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Vowpal Wabbit","items":[{"type":"link","label":"About","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview","docId":"Explore Algorithms/Vowpal Wabbit/Overview"},{"type":"link","label":"Multi-class classification","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification","docId":"Explore Algorithms/Vowpal Wabbit/Multi-class classification"},{"type":"link","label":"Contextual Bandits","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","docId":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits"},{"type":"link","label":"Quickstart - Classification, Quantile Regression, and Regression","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression"},{"type":"link","label":"Quickstart - Classification using SparkML Vectors","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors"},{"type":"link","label":"Quickstart - Classification using VW-native Format","href":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","docId":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Other Algorithms","items":[{"type":"link","label":"SAR Algorithm","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","docId":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations"},{"type":"link","label":"CyberML","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML","docId":"Explore Algorithms/Other Algorithms/Cyber ML"},{"type":"link","label":"Quickstart - Anomalous Access Detection","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","docId":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection"},{"type":"link","label":"Quickstart - Exploring Art Across Cultures","href":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","docId":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures"}],"collapsed":true,"collapsible":true}],"collapsed":true,"collapsible":true},{"type":"category","label":"Use with MLFlow","items":[{"type":"link","label":"Overview","href":"/SynapseML/docs/next/Use with MLFlow/Overview","docId":"Use with MLFlow/Overview"},{"type":"link","label":"Install","href":"/SynapseML/docs/next/Use with MLFlow/Install","docId":"Use with MLFlow/Install"},{"type":"link","label":"Autologging","href":"/SynapseML/docs/next/Use with MLFlow/Autologging","docId":"Use with MLFlow/Autologging"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Deploy Models","items":[{"type":"link","label":"About","href":"/SynapseML/docs/next/Deploy Models/Overview","docId":"Deploy Models/Overview"},{"type":"link","label":"Quickstart - Deploying a Classifier","href":"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier","docId":"Deploy Models/Quickstart - Deploying a Classifier"}],"collapsed":true,"collapsible":true},{"type":"category","label":"Reference","items":[{"type":"link","label":"Contributor Guide","href":"/SynapseML/docs/next/Reference/Contributor Guide","docId":"Reference/Contributor Guide"},{"type":"link","label":"Developer Setup","href":"/SynapseML/docs/next/Reference/Developer Setup","docId":"Reference/Developer Setup"},{"type":"link","label":"Docker Setup","href":"/SynapseML/docs/next/Reference/Docker Setup","docId":"Reference/Docker Setup"},{"type":"link","label":"R setup","href":"/SynapseML/docs/next/Reference/R Setup","docId":"Reference/R Setup"}],"collapsed":true,"collapsible":true}]},"docs":{"Deploy Models/Overview":{"id":"Deploy Models/Overview","title":"Spark Serving","description":"An Engine for Deploying Spark Jobs as Distributed Web Services","sidebar":"docs"},"Deploy Models/Quickstart - Deploying a Classifier":{"id":"Deploy Models/Quickstart - Deploying a Classifier","title":"Quickstart - Deploying a Classifier","description":"Model Deployment with Spark Serving","sidebar":"docs"},"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key":{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","title":"Advanced Usage - Async, Batching, and Multi-Key","description":"Step 1: Imports and Keys","sidebar":"docs"},"Explore Algorithms/AI Services/Geospatial Services":{"id":"Explore Algorithms/AI Services/Geospatial Services","title":"Geospatial Services","description":"Microsoft Azure Maps  provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.","sidebar":"docs"},"Explore Algorithms/AI Services/Multivariate Anomaly Detection":{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","title":"Multivariate Anomaly Detection","description":"This recipe shows how you can use SynapseML and Azure AI services on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI services, and we then use to the model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.","sidebar":"docs"},"Explore Algorithms/AI Services/Overview":{"id":"Explore Algorithms/AI Services/Overview","title":"Overview","description":"Azure AI services help developers and organizations rapidly create intelligent, cutting-edge, market-ready, and responsible applications with out-of-the-box and pre-built and customizable APIs and models.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes":{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","title":"Quickstart - Analyze Celebrity Quotes","description":"Extracting celebrity quote images using Bing Image Search on Spark","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Analyze Text":{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","title":"Quickstart - Analyze Text","description":"Azure AI Language is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine":{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","title":"Quickstart - Create a Visual Search Engine","description":"Creating a searchable Art Database with The MET\'s open-access collection","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Create Audiobooks":{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","title":"Quickstart - Create Audiobooks","description":"Step 1: Load libraries and add service information","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs":{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","title":"Quickstart - Document Question and Answering with PDFs","description":"Introduction","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Flooding Risk":{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","title":"Quickstart - Flooding Risk","description":"King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.","sidebar":"docs"},"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance":{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","title":"Quickstart - Predictive Maintenance","description":"This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We\'ll follow along with the CosmosDB and Synapse Link sample. To keep things simple, in this recipe we\'ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.","sidebar":"docs"},"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests":{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","title":"Quickstart - Isolation Forests","description":"This article shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Overview":{"id":"Explore Algorithms/Causal Inference/Overview","title":"Overview","description":"Causal Inference on Apache Spark","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects":{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","title":"Quickstart - Measure Causal Effects","description":"This sample notebook aims to show the application of using SynapseML\'s DoubleMLEstimator for inferring causality using observational data.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects":{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","title":"Quickstart - Measure Heterogeneous Effects","description":"This sample notebook aims to show the application of using SynapseML\'s DoubleMLEstimator for inferring causality using observational data.","sidebar":"docs"},"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences":{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","title":"Quickstart - Synthetic difference in differences","description":"This sample notebook aims to show readers how to use SynapseML\'s DiffInDiffEstimator, SyntheticControlEstimator and SyntheticDiffInDiffEstimator to estimate the causal effect of a treatment on a particular outcome.","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML":{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","title":"Quickstart - SparkML vs SynapseML","description":"In this article, you perform the same classification task in two","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Train Classifier":{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","title":"Quickstart - Train Classifier","description":"Classification - Adult Census","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data":{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","title":"Quickstart - Vowpal Wabbit on Tabular Data","description":"In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) classifier in SynapseML.","sidebar":"docs"},"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data":{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","title":"Quickstart - Vowpal Wabbit on Text Data","description":"In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is Sentiment140 twitter data. First, we import a few packages that we need.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Distributed Training":{"id":"Explore Algorithms/Deep Learning/Distributed Training","title":"Distributed Training","description":"Why Simple Deep Learning","sidebar":"docs"},"Explore Algorithms/Deep Learning/Getting Started":{"id":"Explore Algorithms/Deep Learning/Getting Started","title":"Getting Started","description":"This is a sample with databricks 10.4.x-gpu-ml-scala2.12 runtime","sidebar":"docs"},"Explore Algorithms/Deep Learning/ONNX":{"id":"Explore Algorithms/Deep Learning/ONNX","title":"ONNX","description":"Learn how to use the ONNX model transformer to run inference for an ONNX model on Spark.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier":{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","title":"Quickstart - Fine-tune a Text Classifier","description":"Deep Learning - Deep Text Classifier","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier":{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","title":"Quickstart - Fine-tune a Vision Classifier","description":"Deep Learning - Deep Vision Classifier","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference":{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","title":"Quickstart - ONNX Model Inference","description":"In this example, you train a LightGBM model and convert the model to ONNX format. Once converted, you use the model to infer some testing data on Spark.","sidebar":"docs"},"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification":{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","title":"Quickstart - Transfer Learn for Image Classification","description":"Deep Learning - Flower Image Classification","sidebar":"docs"},"Explore Algorithms/Hyperparameter Tuning/HyperOpt":{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","title":"HyperOpt","description":"SynapseML is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others.","sidebar":"docs"},"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search":{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","title":"Quickstart - Random Search","description":"This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we\'ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer.","sidebar":"docs"},"Explore Algorithms/LightGBM/Overview":{"id":"Explore Algorithms/LightGBM/Overview","title":"Overview","description":"LightGBM","sidebar":"docs"},"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression":{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","title":"Quickstart - Classification, Ranking, and Regression","description":"What is LightGBM","sidebar":"docs"},"Explore Algorithms/OpenAI/Langchain":{"id":"Explore Algorithms/OpenAI/Langchain","title":"Langchain","description":"LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.","sidebar":"docs"},"Explore Algorithms/OpenAI/OpenAI":{"id":"Explore Algorithms/OpenAI/OpenAI","title":"OpenAI","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure OpenAI.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding":{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","title":"Quickstart - OpenAI Embedding","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN":{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","title":"Quickstart - OpenAI Embedding and GPU based KNN","description":"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library Spark Rapids ML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on \\"Quickstart - OpenAI Embedding\\" notebook with NVIDIA GPU accelerated KNN.","sidebar":"docs"},"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms":{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","title":"Quickstart - Understand and Search Forms","description":"In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:","sidebar":"docs"},"Explore Algorithms/OpenCV/Image Transformations":{"id":"Explore Algorithms/OpenCV/Image Transformations","title":"Image Transformations","description":"OpenCV - Pipeline Image Transformations","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Cyber ML":{"id":"Explore Algorithms/Other Algorithms/Cyber ML","title":"CyberML","description":"access anomalies: complementaccess.py","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection":{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","title":"Quickstart - Anomalous Access Detection","description":"Here we demonstrate a novel CyberML model which can learn user access patterns and then automatically detect anomalous user access based on learned behavior.","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures":{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","title":"Quickstart - Exploring Art Across Cultures","description":"This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.","sidebar":"docs"},"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations":{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","title":"Smart Adaptive Recommendations (SAR) Algorithm","description":"The following document is a subset of the implemented logic. The original can be found here","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Data Cleaning":{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","title":"Quickstart - Data Cleaning","description":"Regression -  Flight Delays with DataCleaning","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Train Regressor":{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","title":"Quickstart - Train Regressor","description":"Regression - Auto Imports","sidebar":"docs"},"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM":{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","title":"Quickstart - Vowpal Wabbit and LightGBM","description":"This notebook shows how to build simple regression models by using","sidebar":"docs"},"Explore Algorithms/Responsible AI/Data Balance Analysis":{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","title":"Data Balance Analysis on Spark","description":"Learn how to do Data Balance Analysis on Spark to determine how well features and feature values are represented in your dataset.","sidebar":"docs"},"Explore Algorithms/Responsible AI/Explanation Dashboard":{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","title":"Explanation Dashboard","description":"Interpretability - Explanation Dashboard","sidebar":"docs"},"Explore Algorithms/Responsible AI/Image Explainers":{"id":"Explore Algorithms/Responsible AI/Image Explainers","title":"Image Explainers","description":"Interpretability - Image Explainers","sidebar":"docs"},"Explore Algorithms/Responsible AI/Interpreting Model Predictions":{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","title":"Interpreting Model Predictions","description":"Interpretable Machine Learning","sidebar":"docs"},"Explore Algorithms/Responsible AI/PDP and ICE Explainers":{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","title":"PDP and ICE Explainers","description":"Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots","sidebar":"docs"},"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis":{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","title":"Quickstart - Data Balance Analysis","description":"Data Balance Analysis using the Adult Census Income dataset","sidebar":"docs"},"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection":{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","title":"Quickstart - Snow Leopard Detection","description":"Automated Snow Leopard Detection with Synapse Machine Learning","sidebar":"docs"},"Explore Algorithms/Responsible AI/Tabular Explainers":{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","title":"Tabular Explainers","description":"In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.","sidebar":"docs"},"Explore Algorithms/Responsible AI/Text Explainers":{"id":"Explore Algorithms/Responsible AI/Text Explainers","title":"Text Explainers","description":"Interpretability - Text Explainers","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Contextual Bandits":{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","title":"Contextual Bandits","description":"In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Multi-class classification":{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","title":"Multi-class classification","description":"Read dataset","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Overview":{"id":"Explore Algorithms/Vowpal Wabbit/Overview","title":"VW","description":"Overview","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","title":"Quickstart - Classification using SparkML Vectors","description":"SparkML Vector input","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","title":"Quickstart - Classification using VW-native Format","description":"SparkML Vector input","sidebar":"docs"},"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression":{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","title":"Quickstart - Classification, Quantile Regression, and Regression","description":"VowpalWabbit (VW) is a machine learning system which","sidebar":"docs"},"Get Started/Create a Spark Cluster":{"id":"Get Started/Create a Spark Cluster","title":"Create a Spark Cluster","description":"SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.","sidebar":"docs"},"Get Started/Install SynapseML":{"id":"Get Started/Install SynapseML","title":"Install SynapseML","description":"Install SynapseML","sidebar":"docs"},"Get Started/Quickstart - Your First Models":{"id":"Get Started/Quickstart - Your First Models","title":"Quickstart - Your First Models","description":"This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.","sidebar":"docs"},"Get Started/Set up Cognitive Services":{"id":"Get Started/Set up Cognitive Services","title":"Set up Cognitive Services","description":"In order to use SynapseML\'s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.","sidebar":"docs"},"Overview":{"id":"Overview","title":"What is SynapseML?","description":"SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the Apache Spark distributed computing framework and shares the same API as the SparkML/MLLib library, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.","sidebar":"docs"},"Quick Examples/estimators/estimators_causal":{"id":"Quick Examples/estimators/estimators_causal","title":"Estimators - Causal","description":""},"Quick Examples/estimators/estimators_cognitive":{"id":"Quick Examples/estimators/estimators_cognitive","title":"Estimators - Cognitive","description":""},"Quick Examples/estimators/estimators_core":{"id":"Quick Examples/estimators/estimators_core","title":"Estimators - Core","description":""},"Quick Examples/estimators/estimators_lightgbm":{"id":"Quick Examples/estimators/estimators_lightgbm","title":"Estimators - LightGBM","description":""},"Quick Examples/estimators/estimators_vw":{"id":"Quick Examples/estimators/estimators_vw","title":"Estimators - Vowpal Wabbit","description":""},"Quick Examples/transformers/transformers_cognitive":{"id":"Quick Examples/transformers/transformers_cognitive","title":"Transformers - Cognitive","description":""},"Quick Examples/transformers/transformers_core":{"id":"Quick Examples/transformers/transformers_core","title":"Transformers - Core","description":""},"Quick Examples/transformers/transformers_deep_learning":{"id":"Quick Examples/transformers/transformers_deep_learning","title":"Deep Learning","description":""},"Quick Examples/transformers/transformers_opencv":{"id":"Quick Examples/transformers/transformers_opencv","title":"Transformers - OpenCV","description":""},"Quick Examples/transformers/transformers_vw":{"id":"Quick Examples/transformers/transformers_vw","title":"Transformers - Vowpal Wabbit","description":""},"Reference/Contributor Guide":{"id":"Reference/Contributor Guide","title":"Contributor Guide","description":"Contributor Guide","sidebar":"docs"},"Reference/Developer Setup":{"id":"Reference/Developer Setup","title":"Developer Setup","description":"Developer Setup","sidebar":"docs"},"Reference/Docker Setup":{"id":"Reference/Docker Setup","title":"Docker Setup","description":"Docker Setup","sidebar":"docs"},"Reference/Dotnet Setup":{"id":"Reference/Dotnet Setup","title":".NET setup","description":".NET setup"},"Reference/Quickstart - LightGBM in Dotnet":{"id":"Reference/Quickstart - LightGBM in Dotnet","title":"Quickstart - LightGBM in Dotnet","description":"A simple example about classification with LightGBMClassifier using .NET"},"Reference/R Setup":{"id":"Reference/R Setup","title":"R setup","description":"R setup and example for SynapseML","sidebar":"docs"},"Use with MLFlow/Autologging":{"id":"Use with MLFlow/Autologging","title":"Autologging","description":"SynapseML autologging","sidebar":"docs"},"Use with MLFlow/Install":{"id":"Use with MLFlow/Install","title":"Install","description":"install Mlflow on different environments","sidebar":"docs"},"Use with MLFlow/Overview":{"id":"Use with MLFlow/Overview","title":"Overview","description":"MLflow support of SynapseML","sidebar":"docs"}}}')}}]);
\ No newline at end of file
diff --git a/assets/js/dd3f9ada.7a974bf8.js b/assets/js/dd3f9ada.7a974bf8.js
deleted file mode 100644
index bd3359ae35..0000000000
--- a/assets/js/dd3f9ada.7a974bf8.js
+++ /dev/null
@@ -1 +0,0 @@
-"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[17820],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>u});var r=n(67294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=p(n),u=a,g=c["".concat(l,".").concat(u)]||c[u]||m[u]||o;return n?r.createElement(g,i(i({ref:t},d),{},{components:n})):r.createElement(g,i({ref:t},d))}));function u(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p<o;p++)i[p]=n[p];return r.createElement.apply(null,i)}return r.createElement.apply(null,n)}c.displayName="MDXCreateElement"},66851:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>s,toc:()=>p});var r=n(83117),a=(n(67294),n(3905));const o={title:"Quickstart - OpenAI Embedding and GPU based KNN",hide_title:!0,status:"stable"},i="Embedding Text with Azure OpenAI and GPU based KNN",s={unversionedId:"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",id:"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",title:"Quickstart - OpenAI Embedding and GPU based KNN",description:'The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library Spark Rapids ML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on "Quickstart - OpenAI Embedding" notebook with NVIDIA GPU accelerated KNN.',source:"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",sourceDirName:"Explore Algorithms/OpenAI",slug:"/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",draft:!1,tags:[],version:"current",frontMatter:{title:"Quickstart - OpenAI Embedding and GPU based KNN",hide_title:!0,status:"stable"},sidebar:"docs",previous:{title:"Quickstart - OpenAI Embedding",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding"},next:{title:"Quickstart - Understand and Search Forms",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms"}},l={},p=[{value:"Step 1: Prerequisites",id:"step-1-prerequisites",level:2},{value:"Step 2: Import this guide as a notebook",id:"step-2-import-this-guide-as-a-notebook",level:2},{value:"Step 3: Fill in your service information",id:"step-3-fill-in-your-service-information",level:2},{value:"Step 4: Load Data",id:"step-4-load-data",level:2},{value:"Step 5: Generate Embeddings",id:"step-5-generate-embeddings",level:2},{value:"Step 6: Reduce Embedding dimensionality for Visualization",id:"step-6-reduce-embedding-dimensionality-for-visualization",level:2},{value:"Step 7: Plot the embeddings",id:"step-7-plot-the-embeddings",level:2},{value:"Step 8: Build the query against embeddings",id:"step-8-build-the-query-against-embeddings",level:2},{value:"Step 9: Fit KNN model",id:"step-9-fit-knn-model",level:2},{value:"Step 10: Retrieve query results",id:"step-10-retrieve-query-results",level:2}],d={toc:p};function m(e){let{components:t,...n}=e;return(0,a.kt)("wrapper",(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"embedding-text-with-azure-openai-and-gpu-based-knn"},"Embedding Text with Azure OpenAI and GPU based KNN"),(0,a.kt)("p",null,"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/NVIDIA/spark-rapids-ml/"},"Spark Rapids ML"),". This integration makes it easy to use the ",(0,a.kt)("a",{parentName:"p",href:"https://spark.apache.org/"},"Apache Spark"),' distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on "Quickstart - OpenAI Embedding" notebook with NVIDIA GPU accelerated KNN.'),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Note"),": Running the notebook with the demo dataset (Step 4) will generate the same results as CPU based \u201cQuickstart - OpenAI Embedding\u201d notebook. To see  GPU acceleration you need to run query against bigger embeddings.\nFor example, running 100K rows dataset will give 6x acceleration and consume less than 10x memory on 2 nodes NVIDIA T4 cluster compare to AMD Epic (Rome) 2 nodes CPU cluster."),(0,a.kt)("h2",{id:"step-1-prerequisites"},"Step 1: Prerequisites"),(0,a.kt)("p",null,"The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the ",(0,a.kt)("inlineCode",{parentName:"p"},"pyspark")," package will work. "),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},"An Azure OpenAI resource \u2013 request access ",(0,a.kt)("a",{parentName:"li",href:"https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu"},"here")," before ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource"},"creating a resource")),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace"},"Create a Synapse workspace")),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool"},"Create a serverless Apache Spark pool"))),(0,a.kt)("h2",{id:"step-2-import-this-guide-as-a-notebook"},"Step 2: Import this guide as a notebook"),(0,a.kt)("p",null,"The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics"),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb"},"Download this demo as a notebook")," (click Raw, then save the file)"),(0,a.kt)("li",{parentName:"ol"},"Import the notebook ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook"},"into the Synapse Workspace")," or if using Databricks ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook"},"into the Databricks Workspace")),(0,a.kt)("li",{parentName:"ol"},"Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of ",(0,a.kt)("a",{parentName:"li",href:"https://microsoft.github.io/SynapseML/"},"the SynapseML website"),". Note that this requires pasting an additional cell at the top of the notebook you just imported"),(0,a.kt)("li",{parentName:"ol"},"To run the notebook on Databricks add related init script (/tools/init_scripts/init-rapidsml-cuda-11.8.sh)"),(0,a.kt)("li",{parentName:"ol"},"Connect your notebook to a cluster and follow along, editing and rnnung the cells below.")),(0,a.kt)("h2",{id:"step-3-fill-in-your-service-information"},"Step 3: Fill in your service information"),(0,a.kt)("p",null,"Next, please edit the cell in the notebook to point to your service. In particular set the ",(0,a.kt)("inlineCode",{parentName:"p"},"service_name"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"deployment_name"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"location"),", and ",(0,a.kt)("inlineCode",{parentName:"p"},"key")," variables to match those for your OpenAI service"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.core.platform import find_secret\n\n# Fill in the following lines with your service information\n# Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model\nservice_name = "synapseml-openai-2"\ndeployment_name_embeddings = "text-embedding-ada-002"\n\nkey = find_secret(\n    secret_name="openai-api-key-2", keyvault="mmlspark-build-keys"\n)  # please replace this with your key as a string\n\nassert key is not None and service_name is not None\n')),(0,a.kt)("h2",{id:"step-4-load-data"},"Step 4: Load Data"),(0,a.kt)("p",null,"In this demo we will explore a dataset of fine food reviews"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import pyspark.sql.functions as F\n\ndf = (\n    spark.read.options(inferSchema="True", delimiter=",", header=True)\n    .csv("wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv")\n    .repartition(5)\n)\n\ndf = df.withColumn(\n    "combined",\n    F.format_string("Title: %s; Content: %s", F.trim(df.Summary), F.trim(df.Text)),\n)\n\ndisplay(df)\n')),(0,a.kt)("h2",{id:"step-5-generate-embeddings"},"Step 5: Generate Embeddings"),(0,a.kt)("p",null,"We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.openai import OpenAIEmbedding\n\nembedding = (\n    OpenAIEmbedding()\n    .setSubscriptionKey(key)\n    .setDeploymentName(deployment_name_embeddings)\n    .setCustomServiceName(service_name)\n    .setTextCol("combined")\n    .setErrorCol("error")\n    .setOutputCol("embeddings")\n)\n\ncompleted_df = embedding.transform(df).cache()\ndisplay(completed_df)\n')),(0,a.kt)("h2",{id:"step-6-reduce-embedding-dimensionality-for-visualization"},"Step 6: Reduce Embedding dimensionality for Visualization"),(0,a.kt)("p",null,"We reduce the dimensionality to 2 dimensions using t-SNE decomposition."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import pandas as pd\nfrom sklearn.manifold import TSNE\nimport numpy as np\n\ncollected = list(completed_df.collect())\nmatrix = np.array([[r["embeddings"]] for r in collected])[:, 0, :].astype(np.float64)\nscores = np.array([[r["Score"]] for r in collected]).reshape(-1)\n\ntsne = TSNE(n_components=2, perplexity=15, random_state=42, init="pca")\nvis_dims = tsne.fit_transform(matrix)\nvis_dims.shape\n')),(0,a.kt)("h2",{id:"step-7-plot-the-embeddings"},"Step 7: Plot the embeddings"),(0,a.kt)("p",null,"We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import matplotlib.pyplot as plt\nimport matplotlib\nimport numpy as np\n\ncolors = ["red", "darkorange", "gold", "turquoise", "darkgreen"]\nx = [x for x, y in vis_dims]\ny = [y for x, y in vis_dims]\ncolor_indices = scores - 1\n\ncolormap = matplotlib.colors.ListedColormap(colors)\nplt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3)\nfor score in [0, 1, 2, 3, 4]:\n    avg_x = np.array(x)[scores - 1 == score].mean()\n    avg_y = np.array(y)[scores - 1 == score].mean()\n    color = colors[score]\n    plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)\n\nplt.title("Amazon ratings visualized in language using t-SNE")\n')),(0,a.kt)("h2",{id:"step-8-build-the-query-against-embeddings"},"Step 8: Build the query against embeddings"),(0,a.kt)("p",null,'Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the "-doc" and "-query" suffixes respectively.'),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.ml import PipelineModel\n\nembedding_query = (\n    OpenAIEmbedding()\n    .setSubscriptionKey(key)\n    .setDeploymentName(deployment_name_embeddings)\n    .setCustomServiceName(service_name)\n    .setTextCol("query")\n    .setErrorCol("error")\n    .setOutputCol("embeddings")\n)\n\nquery_df = (\n    spark.createDataFrame(\n        [\n            (\n                0,\n                "desserts",\n            ),\n            (\n                1,\n                "disgusting",\n            ),\n        ]\n    )\n    .toDF("id", "query")\n    .withColumn("id", F.col("id").cast("int"))\n)\n\nembedding_query_df = (\n    embedding_query.transform(query_df).select("id", "embeddings").cache()\n)\n')),(0,a.kt)("h2",{id:"step-9-fit-knn-model"},"Step 9: Fit KNN model"),(0,a.kt)("p",null,"Build KNN model using fit method"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from spark_rapids_ml.knn import NearestNeighbors\n\nrapids_knn = NearestNeighbors(k=10)\nrapids_knn.setInputCol("embeddings").setIdCol("id")\n\nrapids_knn_model = rapids_knn.fit(completed_df.select("id", "embeddings"))\n')),(0,a.kt)("h2",{id:"step-10-retrieve-query-results"},"Step 10: Retrieve query results"),(0,a.kt)("p",null,"Find k nearest neighbors"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"(_, _, knn_df) = rapids_knn_model.kneighbors(embedding_query_df)\n\ndisplay(knn_df)\n")))}m.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/dd3f9ada.97671d28.js b/assets/js/dd3f9ada.97671d28.js
new file mode 100644
index 0000000000..f8d6fafdaa
--- /dev/null
+++ b/assets/js/dd3f9ada.97671d28.js
@@ -0,0 +1 @@
+"use strict";(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[17820],{3905:(e,t,n)=>{n.d(t,{Zo:()=>d,kt:()=>u});var r=n(67294);function a(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function o(e,t){var n=Object.keys(e);if(Object.getOwnPropertySymbols){var r=Object.getOwnPropertySymbols(e);t&&(r=r.filter((function(t){return Object.getOwnPropertyDescriptor(e,t).enumerable}))),n.push.apply(n,r)}return n}function i(e){for(var t=1;t<arguments.length;t++){var n=null!=arguments[t]?arguments[t]:{};t%2?o(Object(n),!0).forEach((function(t){a(e,t,n[t])})):Object.getOwnPropertyDescriptors?Object.defineProperties(e,Object.getOwnPropertyDescriptors(n)):o(Object(n)).forEach((function(t){Object.defineProperty(e,t,Object.getOwnPropertyDescriptor(n,t))}))}return e}function s(e,t){if(null==e)return{};var n,r,a=function(e,t){if(null==e)return{};var n,r,a={},o=Object.keys(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||(a[n]=e[n]);return a}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(r=0;r<o.length;r++)n=o[r],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(a[n]=e[n])}return a}var l=r.createContext({}),p=function(e){var t=r.useContext(l),n=t;return e&&(n="function"==typeof e?e(t):i(i({},t),e)),n},d=function(e){var t=p(e.components);return r.createElement(l.Provider,{value:t},e.children)},m={inlineCode:"code",wrapper:function(e){var t=e.children;return r.createElement(r.Fragment,{},t)}},c=r.forwardRef((function(e,t){var n=e.components,a=e.mdxType,o=e.originalType,l=e.parentName,d=s(e,["components","mdxType","originalType","parentName"]),c=p(n),u=a,g=c["".concat(l,".").concat(u)]||c[u]||m[u]||o;return n?r.createElement(g,i(i({ref:t},d),{},{components:n})):r.createElement(g,i({ref:t},d))}));function u(e,t){var n=arguments,a=t&&t.mdxType;if("string"==typeof e||a){var o=n.length,i=new Array(o);i[0]=c;var s={};for(var l in t)hasOwnProperty.call(t,l)&&(s[l]=t[l]);s.originalType=e,s.mdxType="string"==typeof e?e:a,i[1]=s;for(var p=2;p<o;p++)i[p]=n[p];return r.createElement.apply(null,i)}return r.createElement.apply(null,n)}c.displayName="MDXCreateElement"},66851:(e,t,n)=>{n.r(t),n.d(t,{assets:()=>l,contentTitle:()=>i,default:()=>m,frontMatter:()=>o,metadata:()=>s,toc:()=>p});var r=n(83117),a=(n(67294),n(3905));const o={title:"Quickstart - OpenAI Embedding and GPU based KNN",hide_title:!0,status:"stable"},i="Embedding Text with Azure OpenAI and GPU based KNN",s={unversionedId:"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",id:"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",title:"Quickstart - OpenAI Embedding and GPU based KNN",description:'The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library Spark Rapids ML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on "Quickstart - OpenAI Embedding" notebook with NVIDIA GPU accelerated KNN.',source:"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",sourceDirName:"Explore Algorithms/OpenAI",slug:"/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN",draft:!1,tags:[],version:"current",frontMatter:{title:"Quickstart - OpenAI Embedding and GPU based KNN",hide_title:!0,status:"stable"},sidebar:"docs",previous:{title:"Quickstart - OpenAI Embedding",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding"},next:{title:"Quickstart - Custom Embeddings and Approximate KNN on GPU",permalink:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU"}},l={},p=[{value:"Step 1: Prerequisites",id:"step-1-prerequisites",level:2},{value:"Step 2: Import this guide as a notebook",id:"step-2-import-this-guide-as-a-notebook",level:2},{value:"Step 3: Fill in your service information",id:"step-3-fill-in-your-service-information",level:2},{value:"Step 4: Load Data",id:"step-4-load-data",level:2},{value:"Step 5: Generate Embeddings",id:"step-5-generate-embeddings",level:2},{value:"Step 6: Reduce Embedding dimensionality for Visualization",id:"step-6-reduce-embedding-dimensionality-for-visualization",level:2},{value:"Step 7: Plot the embeddings",id:"step-7-plot-the-embeddings",level:2},{value:"Step 8: Build the query against embeddings",id:"step-8-build-the-query-against-embeddings",level:2},{value:"Step 9: Fit KNN model",id:"step-9-fit-knn-model",level:2},{value:"Step 10: Retrieve query results",id:"step-10-retrieve-query-results",level:2}],d={toc:p};function m(e){let{components:t,...n}=e;return(0,a.kt)("wrapper",(0,r.Z)({},d,n,{components:t,mdxType:"MDXLayout"}),(0,a.kt)("h1",{id:"embedding-text-with-azure-openai-and-gpu-based-knn"},"Embedding Text with Azure OpenAI and GPU based KNN"),(0,a.kt)("p",null,"The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library ",(0,a.kt)("a",{parentName:"p",href:"https://github.com/NVIDIA/spark-rapids-ml/"},"Spark Rapids ML"),". This integration makes it easy to use the ",(0,a.kt)("a",{parentName:"p",href:"https://spark.apache.org/"},"Apache Spark"),' distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on "Quickstart - OpenAI Embedding" notebook with NVIDIA GPU accelerated KNN.'),(0,a.kt)("p",null,(0,a.kt)("strong",{parentName:"p"},"Note"),": Running the notebook with the demo dataset (Step 4) will generate the same results as CPU based \u201cQuickstart - OpenAI Embedding\u201d notebook. To see  GPU acceleration you need to run query against bigger embeddings.\nFor example, running 100K rows dataset will give 6x acceleration and consume less than 10x memory on 2 nodes NVIDIA T4 cluster compare to AMD Epic (Rome) 2 nodes CPU cluster."),(0,a.kt)("h2",{id:"step-1-prerequisites"},"Step 1: Prerequisites"),(0,a.kt)("p",null,"The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the ",(0,a.kt)("inlineCode",{parentName:"p"},"pyspark")," package will work. "),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},"An Azure OpenAI resource \u2013 request access ",(0,a.kt)("a",{parentName:"li",href:"https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu"},"here")," before ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource"},"creating a resource")),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace"},"Create a Synapse workspace")),(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool"},"Create a serverless Apache Spark pool"))),(0,a.kt)("h2",{id:"step-2-import-this-guide-as-a-notebook"},"Step 2: Import this guide as a notebook"),(0,a.kt)("p",null,"The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics"),(0,a.kt)("ol",null,(0,a.kt)("li",{parentName:"ol"},(0,a.kt)("a",{parentName:"li",href:"https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb"},"Download this demo as a notebook")," (click Raw, then save the file)"),(0,a.kt)("li",{parentName:"ol"},"Import the notebook ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook"},"into the Synapse Workspace")," or if using Databricks ",(0,a.kt)("a",{parentName:"li",href:"https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook"},"into the Databricks Workspace")),(0,a.kt)("li",{parentName:"ol"},"Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of ",(0,a.kt)("a",{parentName:"li",href:"https://microsoft.github.io/SynapseML/"},"the SynapseML website"),". Note that this requires pasting an additional cell at the top of the notebook you just imported"),(0,a.kt)("li",{parentName:"ol"},"To run the notebook on Databricks add related init script (/tools/init_scripts/init-rapidsml-cuda-11.8.sh)"),(0,a.kt)("li",{parentName:"ol"},"Connect your notebook to a cluster and follow along, editing and rnnung the cells below.")),(0,a.kt)("h2",{id:"step-3-fill-in-your-service-information"},"Step 3: Fill in your service information"),(0,a.kt)("p",null,"Next, please edit the cell in the notebook to point to your service. In particular set the ",(0,a.kt)("inlineCode",{parentName:"p"},"service_name"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"deployment_name"),", ",(0,a.kt)("inlineCode",{parentName:"p"},"location"),", and ",(0,a.kt)("inlineCode",{parentName:"p"},"key")," variables to match those for your OpenAI service"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.core.platform import find_secret\n\n# Fill in the following lines with your service information\n# Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model\nservice_name = "synapseml-openai-2"\ndeployment_name_embeddings = "text-embedding-ada-002"\n\nkey = find_secret(\n    secret_name="openai-api-key-2", keyvault="mmlspark-build-keys"\n)  # please replace this with your key as a string\n\nassert key is not None and service_name is not None\n')),(0,a.kt)("h2",{id:"step-4-load-data"},"Step 4: Load Data"),(0,a.kt)("p",null,"In this demo we will explore a dataset of fine food reviews"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import pyspark.sql.functions as F\n\ndf = (\n    spark.read.options(inferSchema="True", delimiter=",", header=True)\n    .csv("wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv")\n    .repartition(5)\n)\n\ndf = df.withColumn(\n    "combined",\n    F.format_string("Title: %s; Content: %s", F.trim(df.Summary), F.trim(df.Text)),\n)\n\ndisplay(df)\n')),(0,a.kt)("h2",{id:"step-5-generate-embeddings"},"Step 5: Generate Embeddings"),(0,a.kt)("p",null,"We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from synapse.ml.services.openai import OpenAIEmbedding\n\nembedding = (\n    OpenAIEmbedding()\n    .setSubscriptionKey(key)\n    .setDeploymentName(deployment_name_embeddings)\n    .setCustomServiceName(service_name)\n    .setTextCol("combined")\n    .setErrorCol("error")\n    .setOutputCol("embeddings")\n)\n\ncompleted_df = embedding.transform(df).cache()\ndisplay(completed_df)\n')),(0,a.kt)("h2",{id:"step-6-reduce-embedding-dimensionality-for-visualization"},"Step 6: Reduce Embedding dimensionality for Visualization"),(0,a.kt)("p",null,"We reduce the dimensionality to 2 dimensions using t-SNE decomposition."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import pandas as pd\nfrom sklearn.manifold import TSNE\nimport numpy as np\n\ncollected = list(completed_df.collect())\nmatrix = np.array([[r["embeddings"]] for r in collected])[:, 0, :].astype(np.float64)\nscores = np.array([[r["Score"]] for r in collected]).reshape(-1)\n\ntsne = TSNE(n_components=2, perplexity=15, random_state=42, init="pca")\nvis_dims = tsne.fit_transform(matrix)\nvis_dims.shape\n')),(0,a.kt)("h2",{id:"step-7-plot-the-embeddings"},"Step 7: Plot the embeddings"),(0,a.kt)("p",null,"We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions."),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'import matplotlib.pyplot as plt\nimport matplotlib\nimport numpy as np\n\ncolors = ["red", "darkorange", "gold", "turquoise", "darkgreen"]\nx = [x for x, y in vis_dims]\ny = [y for x, y in vis_dims]\ncolor_indices = scores - 1\n\ncolormap = matplotlib.colors.ListedColormap(colors)\nplt.scatter(x, y, c=color_indices, cmap=colormap, alpha=0.3)\nfor score in [0, 1, 2, 3, 4]:\n    avg_x = np.array(x)[scores - 1 == score].mean()\n    avg_y = np.array(y)[scores - 1 == score].mean()\n    color = colors[score]\n    plt.scatter(avg_x, avg_y, marker="x", color=color, s=100)\n\nplt.title("Amazon ratings visualized in language using t-SNE")\n')),(0,a.kt)("h2",{id:"step-8-build-the-query-against-embeddings"},"Step 8: Build the query against embeddings"),(0,a.kt)("p",null,'Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the "-doc" and "-query" suffixes respectively.'),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from pyspark.ml import PipelineModel\n\nembedding_query = (\n    OpenAIEmbedding()\n    .setSubscriptionKey(key)\n    .setDeploymentName(deployment_name_embeddings)\n    .setCustomServiceName(service_name)\n    .setTextCol("query")\n    .setErrorCol("error")\n    .setOutputCol("embeddings")\n)\n\nquery_df = (\n    spark.createDataFrame(\n        [\n            (\n                0,\n                "desserts",\n            ),\n            (\n                1,\n                "disgusting",\n            ),\n        ]\n    )\n    .toDF("id", "query")\n    .withColumn("id", F.col("id").cast("int"))\n)\n\nembedding_query_df = (\n    embedding_query.transform(query_df).select("id", "embeddings").cache()\n)\n')),(0,a.kt)("h2",{id:"step-9-fit-knn-model"},"Step 9: Fit KNN model"),(0,a.kt)("p",null,"Build KNN model using fit method"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},'from spark_rapids_ml.knn import NearestNeighbors\n\nrapids_knn = NearestNeighbors(k=10)\nrapids_knn.setInputCol("embeddings").setIdCol("id")\n\nrapids_knn_model = rapids_knn.fit(completed_df.select("id", "embeddings"))\n')),(0,a.kt)("h2",{id:"step-10-retrieve-query-results"},"Step 10: Retrieve query results"),(0,a.kt)("p",null,"Find k nearest neighbors"),(0,a.kt)("pre",null,(0,a.kt)("code",{parentName:"pre",className:"language-python"},"(_, _, knn_df) = rapids_knn_model.kneighbors(embedding_query_df)\n\ndisplay(knn_df)\n")))}m.isMDXComponent=!0}}]);
\ No newline at end of file
diff --git a/assets/js/main.acd258f6.js b/assets/js/main.acd258f6.js
new file mode 100644
index 0000000000..e98f8341f6
--- /dev/null
+++ b/assets/js/main.acd258f6.js
@@ -0,0 +1,2 @@
+/*! For license information please see main.acd258f6.js.LICENSE.txt */
+(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[40179],{20830:(e,t,n)=>{"use strict";n.d(t,{W:()=>r});var s=n(67294);function r(){return s.createElement("svg",{width:"20",height:"20",className:"DocSearch-Search-Icon",viewBox:"0 0 20 20"},s.createElement("path",{d:"M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z",stroke:"currentColor",fill:"none",fillRule:"evenodd",strokeLinecap:"round",strokeLinejoin:"round"}))}},723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>m});var s=n(67294),r=n(83117),o=n(68356),a=n.n(o),i=n(16887);const c={"00056db7":[()=>n.e(81663).then(n.bind(n,83477)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard.md",83477],"001aea4c":[()=>Promise.all([n.e(40532),n.e(390)]).then(n.bind(n,5642)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_cognitive.md",5642],"00353f68":[()=>n.e(79043).then(n.bind(n,82885)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",82885],"00ab2cda":[()=>n.e(44334).then(n.bind(n,19118)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",19118],"00e0a48e":[()=>n.e(96922).then(n.bind(n,7174)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU.md",7174],"012c1c0f":[()=>Promise.all([n.e(40532),n.e(38261)]).then(n.bind(n,45919)),"@site/versioned_docs/version-1.0.2/Reference/Dotnet Setup.md",45919],"014ce9a3":[()=>n.e(10643).then(n.bind(n,63969)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Data Balance Analysis.md",63969],"01587ade":[()=>n.e(33988).then(n.bind(n,1059)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Distributed Training.md",1059],"01a15f20":[()=>n.e(81970).then(n.bind(n,53471)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",53471],"01eb996b":[()=>n.e(47552).then(n.bind(n,12231)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",12231],"0349e4f0":[()=>n.e(58524).then(n.bind(n,14930)),"@site/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",14930],"034e6169":[()=>n.e(43730).then(n.bind(n,28641)),"@site/versioned_docs/version-0.11.4/Get Started/Install SynapseML.md",28641],"03d0d544":[()=>n.e(26676).then(n.bind(n,46814)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",46814],"0453ae68":[()=>n.e(35939).then(n.bind(n,18496)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",18496],"04804d02":[()=>n.e(41607).then(n.bind(n,17044)),"@site/versioned_docs/version-1.0.4/Reference/R Setup.md",17044],"04ee4ab4":[()=>n.e(62217).then(n.bind(n,41680)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Distributed Training.md",41680],"04fc65cc":[()=>n.e(61755).then(n.bind(n,92078)),"@site/versioned_docs/version-0.11.3/Reference/R Setup.md",92078],"052105dd":[()=>n.e(23036).then(n.bind(n,47333)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard.md",47333],"0606f1a5":[()=>n.e(32789).then(n.bind(n,80)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",80],"061b38c5":[()=>n.e(35124).then(n.bind(n,60062)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",60062],"0637d1e7":[()=>n.e(35750).then(n.bind(n,95457)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",95457],"069e3c4c":[()=>n.e(50745).then(n.bind(n,64206)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",64206],"0713a501":[()=>n.e(37592).then(n.bind(n,9134)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",9134],"077ca2ec":[()=>n.e(37610).then(n.bind(n,9661)),"@site/versioned_docs/version-1.0.1/Reference/Quickstart - LightGBM in Dotnet.md",9661],"07f6b90a":[()=>Promise.all([n.e(40532),n.e(15839)]).then(n.bind(n,55678)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_vw.md",55678],"08019edc":[()=>n.e(62773).then(n.bind(n,55377)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",55377],"093b81f4":[()=>n.e(99970).then(n.bind(n,86786)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",86786],"09990b79":[()=>Promise.all([n.e(40532),n.e(97800)]).then(n.bind(n,88602)),"@site/versioned_docs/version-1.0.3/Reference/Dotnet Setup.md",88602],"09bc1516":[()=>n.e(2408).then(n.bind(n,22601)),"@site/versioned_docs/version-0.11.3/Overview.md",22601],"0a65b9de":[()=>n.e(939).then(n.bind(n,76455)),"@site/versioned_docs/version-0.11.3/Get Started/Set up Cognitive Services.md",76455],"0a954229":[()=>n.e(79088).then(n.bind(n,4489)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",4489],"0a9c6347":[()=>n.e(30460).then(n.bind(n,50140)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",50140],"0b416bde":[()=>n.e(78579).then(n.bind(n,76262)),"@site/versioned_docs/version-0.11.4/Reference/Quickstart - LightGBM in Dotnet.md",76262],"0d432c84":[()=>n.e(34797).then(n.bind(n,72089)),"@site/versioned_docs/version-1.0.1/Get Started/Set up Cognitive Services.md",72089],"0d6d64cd":[()=>Promise.all([n.e(40532),n.e(39307)]).then(n.bind(n,87804)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_cognitive.md",87804],"0daa5b3f":[()=>n.e(35024).then(n.bind(n,73426)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",73426],"0e0ee9b2":[()=>n.e(92045).then(n.bind(n,29119)),"@site/versioned_docs/version-0.11.4/Get Started/Quickstart - Your First Models.md",29119],"0e16e9ea":[()=>n.e(97721).then(n.bind(n,37562)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",37562],"0e2bf69e":[()=>n.e(94395).then(n.bind(n,87433)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",87433],"0e3f998d":[()=>n.e(64482).then(n.bind(n,67479)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",67479],"0eaf8468":[()=>n.e(26184).then(n.bind(n,74137)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",74137],"0f113696":[()=>n.e(16030).then(n.bind(n,78281)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/ONNX.md",78281],"0fa36e32":[()=>n.e(66555).then(n.bind(n,33171)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",33171],"0fb98d33":[()=>n.e(46338).then(n.bind(n,63159)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",63159],"1014102e":[()=>n.e(94744).then(n.bind(n,1110)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Cyber ML.md",1110],"104b9b56":[()=>Promise.all([n.e(40532),n.e(45282)]).then(n.bind(n,90171)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_causal.md",90171],"104bf0a6":[()=>n.e(68314).then(n.bind(n,15734)),"@site/docs/Reference/Contributor Guide.md",15734],"109f6864":[()=>n.e(94994).then(n.bind(n,95820)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",95820],"10b7acf6":[()=>Promise.all([n.e(40532),n.e(6004)]).then(n.bind(n,40044)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_vw.md",40044],"10e13725":[()=>Promise.all([n.e(40532),n.e(25116)]).then(n.bind(n,34855)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_cognitive.md",34855],"1155dc6f":[()=>n.e(5789).then(n.bind(n,74766)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",74766],"11c317cb":[()=>n.e(18933).then(n.bind(n,24433)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Text Explainers.md",24433],"120352d6":[()=>n.e(89262).then(n.bind(n,17608)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",17608],"1293b2b0":[()=>n.e(57074).then(n.bind(n,73303)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier.md",73303],"12b89d4a":[()=>n.e(52804).then(n.bind(n,44536)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Overview.md",44536],"12d85ab5":[()=>n.e(82412).then(n.bind(n,11105)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Overview.md",11105],"13246fd5":[()=>n.e(48807).then(n.bind(n,13869)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Overview.md",13869],"13f17286":[()=>n.e(72912).then(n.bind(n,38060)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",38060],"141dc4a8":[()=>n.e(34334).then(n.bind(n,72356)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/LightGBM/Overview.md",72356],"1518f45a":[()=>n.e(9296).then(n.bind(n,54429)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/OpenAI.md",54429],15516451:[()=>n.e(99128).then(n.bind(n,23312)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Overview.md",23312],"1587ba1d":[()=>n.e(43156).then(n.bind(n,86447)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Overview.md",86447],"15a59c33":[()=>n.e(57924).then(n.bind(n,6296)),"@site/versioned_docs/version-0.11.3/Deploy Models/Overview.md",6296],"15ea8379":[()=>n.e(19483).then(n.bind(n,71915)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",71915],"166d99f5":[()=>n.e(29696).then(n.bind(n,82914)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",82914],"16d05917":[()=>n.e(38759).then(n.bind(n,2727)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",2727],"16d8ca91":[()=>n.e(39402).then(n.bind(n,2422)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",2422],"1766722a":[()=>n.e(17817).then(n.bind(n,97592)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",97592],17896441:[()=>Promise.all([n.e(40532),n.e(24572),n.e(27918)]).then(n.bind(n,78945)),"@theme/DocItem",78945],"1789daeb":[()=>n.e(42377).then(n.bind(n,86544)),"@site/versioned_docs/version-1.0.2/Reference/Docker Setup.md",86544],"179d57fa":[()=>n.e(71580).then(n.bind(n,59171)),"@site/versioned_docs/version-1.0.1/Deploy Models/Quickstart - Deploying a Classifier.md",59171],"17f418b7":[()=>n.e(80837).then(n.bind(n,84998)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Install.md",84998],"1842239f":[()=>Promise.all([n.e(40532),n.e(14535)]).then(n.bind(n,3314)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_vw.md",3314],"18c7d2a3":[()=>n.e(15800).then(n.bind(n,62323)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",62323],"18c9c2ba":[()=>n.e(68585).then(n.bind(n,89326)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard.md",89326],"1a215693":[()=>n.e(71535).then(n.bind(n,85042)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",85042],"1a3608a9":[()=>n.e(24484).then(n.bind(n,75661)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",75661],"1a4e3797":[()=>Promise.all([n.e(40532),n.e(97920)]).then(n.bind(n,39172)),"@theme/SearchPage",39172],"1a62d1af":[()=>n.e(96960).then(n.bind(n,26408)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",26408],"1abf1e18":[()=>n.e(15343).then(n.bind(n,57819)),"@site/blog/2019-08-24-Welcome to Azure Cognitive Services.md",57819],"1ad91055":[()=>n.e(4193).then(n.bind(n,81719)),"@site/versioned_docs/version-1.0.1/Overview.md",81719],"1b3ab3b8":[()=>Promise.all([n.e(40532),n.e(13431)]).then(n.bind(n,53630)),"@site/docs/Quick Examples/transformers/transformers_cognitive.md",53630],"1b7af47d":[()=>n.e(14273).then(n.bind(n,51611)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",51611],"1be78505":[()=>Promise.all([n.e(40532),n.e(29514)]).then(n.bind(n,19963)),"@theme/DocPage",19963],"1d820955":[()=>n.e(81913).then(n.bind(n,58442)),"@site/docs/Explore Algorithms/LightGBM/Overview.md",58442],"1db1f785":[()=>n.e(40950).then(n.bind(n,96420)),"@site/docs/Reference/Developer Setup.md",96420],"1dc2f362":[()=>n.e(89864).then(n.bind(n,69454)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",69454],"1e0d2d3b":[()=>n.e(4503).then(n.bind(n,34824)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",34824],"1e14124f":[()=>n.e(96350).then(n.bind(n,35821)),"@site/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",35821],"1e57c92a":[()=>n.e(36958).then(n.bind(n,87204)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",87204],"1ef65056":[()=>Promise.all([n.e(40532),n.e(72326)]).then(n.bind(n,86903)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_vw.md",86903],"1f31a275":[()=>n.e(94353).then(n.bind(n,53512)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Install.md",53512],"1f6f5ac9":[()=>n.e(1123).then(n.bind(n,9198)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Getting Started.md",9198],20641120:[()=>n.e(787).then(n.bind(n,65432)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",65432],"20e0fe38":[()=>n.e(21102).then(n.bind(n,76411)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Autologging.md",76411],"2137a7cd":[()=>n.e(91391).then(n.bind(n,13295)),"@site/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",13295],"21f47424":[()=>Promise.all([n.e(40532),n.e(37474)]).then(n.bind(n,25305)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_vw.md",25305],"22444eb9":[()=>n.e(30335).then(n.bind(n,16807)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",16807],"22f921cc":[()=>n.e(35517).then(n.bind(n,18751)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",18751],"232d09b0":[()=>n.e(69314).then(n.bind(n,73850)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Getting Started.md",73850],"23af8e3e":[()=>n.e(82723).then(n.bind(n,98894)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Overview.md",98894],"2406587d":[()=>n.e(99119).then(n.bind(n,44519)),"@site/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",44519],"2461af4d":[()=>n.e(70880).then(n.bind(n,29727)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Langchain.md",29727],"255635fc":[()=>n.e(1172).then(n.bind(n,90421)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",90421],"25fb872c":[()=>n.e(48602).then(n.bind(n,502)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Autologging.md",502],26631026:[()=>n.e(78480).then(n.bind(n,22137)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",22137],"268b7b40":[()=>n.e(7234).then(n.bind(n,84922)),"@site/versioned_docs/version-1.0.2/Get Started/Quickstart - Your First Models.md",84922],"275c0efd":[()=>n.e(19950).then(n.bind(n,68567)),"@site/versioned_docs/version-1.0.3/Reference/Contributor Guide.md",68567],"27de0303":[()=>n.e(37334).then(n.bind(n,78849)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",78849],"283b8f99":[()=>n.e(95521).then(n.bind(n,51404)),"@site/docs/Get Started/Quickstart - Your First Models.md",51404],"294c060e":[()=>n.e(97165).then(n.bind(n,58142)),"@site/versioned_docs/version-1.0.4/Reference/Contributor Guide.md",58142],"295a8e78":[()=>n.e(26021).then(n.bind(n,67319)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",67319],"296054a7":[()=>n.e(66945).then(n.bind(n,87853)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",87853],"2a023663":[()=>n.e(97715).then(n.bind(n,23168)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Autologging.md",23168],"2b30bc14":[()=>n.e(19603).then(n.bind(n,12876)),"@site/blog/2019-08-24-Welcome to Azure Cognitive Services.md?truncated=true",12876],"2bcda99e":[()=>n.e(23941).then(n.bind(n,41368)),"@site/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",41368],"2c5f0abd":[()=>n.e(35015).then(n.bind(n,95677)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",95677],"2c98ca3e":[()=>n.e(62106).then(n.bind(n,6036)),"@site/docs/Deploy Models/Quickstart - Deploying a Classifier.md",6036],"2cec6bcf":[()=>n.e(41987).then(n.bind(n,95992)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Cyber ML.md",95992],"2d527871":[()=>n.e(8749).then(n.bind(n,90527)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",90527],"2d5b0e6a":[()=>n.e(28957).then(n.bind(n,33468)),"@site/blog/2018-04-01-Flexible and Scalable Deep Learning with MMLSpark.md?truncated=true",33468],"2d8a44d7":[()=>n.e(44534).then(n.bind(n,79259)),"@site/versioned_docs/version-1.0.3/Get Started/Install SynapseML.md",79259],"2e5f4263":[()=>n.e(52447).then(n.bind(n,18126)),"@site/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",18126],"2e93ae0f":[()=>n.e(40023).then(n.bind(n,60199)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",60199],"2ece0fbe":[()=>n.e(11561).then(n.bind(n,48438)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier.md",48438],"2fb6c1b0":[()=>n.e(80982).then(n.bind(n,76241)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",76241],"318ccb43":[()=>n.e(93377).then(n.bind(n,52979)),"@site/docs/Explore Algorithms/AI Services/Geospatial Services.md",52979],"31bb7274":[()=>n.e(67394).then(n.bind(n,42024)),"@site/docs/Explore Algorithms/Regression/Quickstart - Train Regressor.md",42024],"31ee0fa3":[()=>n.e(42428).then(n.bind(n,28431)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",28431],"3201258e":[()=>n.e(68526).then(n.bind(n,36362)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",36362],"321ba9c5":[()=>n.e(85332).then(n.bind(n,10360)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Geospatial Services.md",10360],"325e22bd":[()=>Promise.all([n.e(40532),n.e(92321)]).then(n.bind(n,23819)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_cognitive.md",23819],"327be84b":[()=>n.e(38399).then(n.bind(n,17630)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",17630],"328f3af4":[()=>n.e(42037).then(n.bind(n,16407)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/ONNX.md",16407],"32fe2e34":[()=>n.e(226).then(n.bind(n,28506)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/OpenAI.md",28506],"336404dc":[()=>n.e(43488).then(n.bind(n,49923)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Overview.md",49923],"33b3776b":[()=>n.e(5254).then(n.bind(n,35734)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",35734],"345903d4":[()=>n.e(52433).then(n.bind(n,31889)),"@site/versioned_docs/version-1.0.1/Reference/Docker Setup.md",31889],"34f00221":[()=>Promise.all([n.e(40532),n.e(37853),n.e(76454)]).then(n.bind(n,63785)),"@site/src/pages/videos.js",63785],"34ff21b6":[()=>Promise.all([n.e(40532),n.e(79362)]).then(n.bind(n,93488)),"@site/docs/Quick Examples/estimators/estimators_cognitive.md",93488],"35826f14":[()=>n.e(23894).then(n.bind(n,88680)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Install.md",88680],"35bdc661":[()=>n.e(27813).then(n.bind(n,58782)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",58782],"3647ac9b":[()=>n.e(92982).then(n.bind(n,66377)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",66377],"368de161":[()=>n.e(1985).then(n.bind(n,83832)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Train Classifier.md",83832],"37ab9beb":[()=>n.e(64992).then(n.bind(n,1050)),"@site/versioned_docs/version-0.11.3/Deploy Models/Quickstart - Deploying a Classifier.md",1050],"38b375cc":[()=>n.e(72249).then(n.bind(n,86440)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",86440],"38d6824e":[()=>n.e(25990).then(n.bind(n,85202)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",85202],"39122aab":[()=>n.e(57689).then(n.bind(n,37155)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",37155],"391cb159":[()=>n.e(41867).then(n.bind(n,55482)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/LightGBM/Overview.md",55482],"39582d99":[()=>Promise.all([n.e(40532),n.e(77697)]).then(n.bind(n,72711)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_opencv.md",72711],"39a50b22":[()=>n.e(29845).then(n.bind(n,13442)),"@site/versioned_docs/version-1.0.3/Overview.md",13442],"3a1147b5":[()=>n.e(82668).then(n.bind(n,92857)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers.md",92857],"3a4f9d93":[()=>n.e(15487).then(n.bind(n,90849)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Getting Started.md",90849],"3a7a88af":[()=>n.e(49814).then(n.bind(n,31377)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",31377],"3ac37a06":[()=>Promise.all([n.e(40532),n.e(91103)]).then(n.bind(n,98105)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_deep_learning.md",98105],"3b5de274":[()=>n.e(11977).then(n.bind(n,69625)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/ONNX.md",69625],"3b75a1c6":[()=>n.e(78047).then(n.bind(n,18615)),"@site/versioned_docs/version-1.0.4/Deploy Models/Overview.md",18615],"3bbe0f34":[()=>n.e(64172).then(n.bind(n,90054)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Image Explainers.md",90054],"3bed3e2f":[()=>n.e(58846).then(n.bind(n,38189)),"@site/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",38189],"3c1f4383":[()=>n.e(16551).then(n.bind(n,52839)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Install.md",52839],"3c22aa66":[()=>n.e(86482).then(n.bind(n,18590)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",18590],"3cbfbf75":[()=>n.e(1773).then(n.bind(n,97312)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor.md",97312],"3d963cd5":[()=>n.e(72286).then(n.bind(n,62991)),"@site/versioned_docs/version-1.0.4/Deploy Models/Quickstart - Deploying a Classifier.md",62991],"3dafa002":[()=>n.e(76258).then(n.bind(n,35440)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",35440],"3e1e84da":[()=>n.e(34397).then(n.bind(n,69813)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/OpenAI.md",69813],"3e7c5569":[()=>n.e(49778).then(n.bind(n,49138)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",49138],"3e9a7422":[()=>Promise.all([n.e(40532),n.e(83393)]).then(n.bind(n,73297)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_vw.md",73297],"3ea19ec0":[()=>n.e(17026).then(n.bind(n,90308)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",90308],"3ee83f58":[()=>n.e(42802).then(n.bind(n,46489)),"@site/versioned_docs/version-1.0.3/Reference/Docker Setup.md",46489],"3ef20ed6":[()=>n.e(61407).then(n.bind(n,2784)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",2784],"3fb29942":[()=>n.e(69559).then(n.t.bind(n,24469,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-blog/default/plugin-route-context-module-100.json",24469],"403e642d":[()=>n.e(60055).then(n.bind(n,5312)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",5312],"405fabd0":[()=>n.e(5253).then(n.bind(n,3354)),"@site/versioned_docs/version-1.0.4/Reference/Docker Setup.md",3354],"408524d4":[()=>Promise.all([n.e(40532),n.e(48007)]).then(n.bind(n,57898)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_cognitive.md",57898],"40b49758":[()=>n.e(37069).then(n.bind(n,33548)),"@site/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",33548],"4130db01":[()=>n.e(86184).then(n.bind(n,40606)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",40606],"41ee0cff":[()=>n.e(55702).then(n.bind(n,5380)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",5380],"41f327fc":[()=>n.e(8966).then(n.bind(n,81631)),"@site/docs/Explore Algorithms/Classification/Quickstart - Train Classifier.md",81631],"421d486c":[()=>Promise.all([n.e(40532),n.e(1084)]).then(n.bind(n,24310)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_core.md",24310],"423540f2":[()=>n.e(42852).then(n.bind(n,80918)),"@site/versioned_docs/version-1.0.1/Get Started/Create a Spark Cluster.md",80918],"42736d5f":[()=>n.e(42947).then(n.bind(n,25348)),"@site/docs/Explore Algorithms/Deep Learning/Getting Started.md",25348],"445012ce":[()=>n.e(3871).then(n.bind(n,56324)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Overview.md",56324],"445a1b28":[()=>Promise.all([n.e(40532),n.e(18373)]).then(n.bind(n,73225)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_lightgbm.md",73225],"44ff0d07":[()=>n.e(65437).then(n.bind(n,62634)),"@site/docs/Reference/R Setup.md",62634],"455aa1bd":[()=>n.e(43251).then(n.bind(n,39068)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Text Explainers.md",39068],"4585b70e":[()=>Promise.all([n.e(40532),n.e(90544)]).then(n.bind(n,53232)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_deep_learning.md",53232],"4631a032":[()=>n.e(37676).then(n.bind(n,25217)),"@site/versioned_docs/version-1.0.4/Reference/Quickstart - LightGBM in Dotnet.md",25217],"47bbce76":[()=>n.e(30464).then(n.bind(n,14008)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",14008],"4801d370":[()=>n.e(59394).then(n.bind(n,58406)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",58406],"48329b94":[()=>Promise.all([n.e(40532),n.e(23979)]).then(n.bind(n,59100)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_deep_learning.md",59100],"48cf7354":[()=>n.e(82994).then(n.bind(n,82516)),"@site/docs/Explore Algorithms/Responsible AI/Tabular Explainers.md",82516],"494501bd":[()=>Promise.all([n.e(40532),n.e(43426)]).then(n.bind(n,41878)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_core.md",41878],"4997ef4f":[()=>n.e(74772).then(n.bind(n,77565)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",77565],"49e6864b":[()=>n.e(22707).then(n.bind(n,52565)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Image Explainers.md",52565],"4b18e5ae":[()=>n.e(38962).then(n.bind(n,16208)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",16208],"4bbbdfcf":[()=>n.e(6413).then(n.bind(n,87267)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",87267],"4bea8531":[()=>n.e(12840).then(n.bind(n,49218)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",49218],"4cee39d7":[()=>n.e(94756).then(n.bind(n,74001)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",74001],"4d31bfb3":[()=>n.e(58092).then(n.bind(n,66388)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",66388],"4db82253":[()=>n.e(97381).then(n.bind(n,45355)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",45355],"4de42975":[()=>n.e(91570).then(n.bind(n,68762)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",68762],"4e044b4e":[()=>Promise.all([n.e(40532),n.e(85847)]).then(n.bind(n,40114)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_lightgbm.md",40114],"4e105cba":[()=>n.e(39596).then(n.bind(n,49164)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",49164],"4e3910af":[()=>n.e(23544).then(n.bind(n,3030)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Cyber ML.md",3030],"4e432063":[()=>n.e(97988).then(n.t.bind(n,43260,19)),"~docs/default/version-0-11-4-metadata-prop-456.json",43260],"4ef2f7cf":[()=>n.e(62675).then(n.bind(n,82055)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",82055],"4f1ca6a9":[()=>n.e(9540).then(n.bind(n,69671)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Overview.md",69671],"4f81391a":[()=>n.e(15165).then(n.bind(n,91769)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",91769],"5066efb2":[()=>n.e(88003).then(n.bind(n,96933)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",96933],"50af03e4":[()=>n.e(36204).then(n.bind(n,79410)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Text Explainers.md",79410],"50f80512":[()=>n.e(77682).then(n.bind(n,90256)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",90256],"513002de":[()=>n.e(1802).then(n.bind(n,76933)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",76933],"513214d5":[()=>Promise.all([n.e(40532),n.e(3701)]).then(n.bind(n,98157)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_opencv.md",98157],"514f485e":[()=>n.e(18828).then(n.bind(n,53701)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",53701],"51878b77":[()=>n.e(77225).then(n.bind(n,6382)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",6382],"51dbb02f":[()=>n.e(16975).then(n.bind(n,11091)),"@site/docs/Explore Algorithms/Responsible AI/Explanation Dashboard.md",11091],"521f9727":[()=>Promise.all([n.e(40532),n.e(31333)]).then(n.bind(n,66202)),"@site/docs/Quick Examples/estimators/estimators_causal.md",66202],"5225b7e0":[()=>n.e(73249).then(n.bind(n,25101)),"@site/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",25101],"52880d18":[()=>n.e(62750).then(n.bind(n,98124)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",98124],"53a5cb1e":[()=>n.e(94548).then(n.bind(n,51847)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor.md",51847],"5407376f":[()=>n.e(22287).then(n.bind(n,87405)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",87405],"54287b47":[()=>n.e(35328).then(n.bind(n,81589)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",81589],"548964ce":[()=>Promise.all([n.e(40532),n.e(19465)]).then(n.bind(n,25947)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_vw.md",25947],55203621:[()=>n.e(3554).then(n.bind(n,40218)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Overview.md",40218],"556ad472":[()=>n.e(80751).then(n.bind(n,43170)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",43170],"56087ddb":[()=>n.e(72293).then(n.bind(n,25952)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",25952],"561bd03d":[()=>n.e(32185).then(n.bind(n,97308)),"@site/blog/2020-12-01-Large-Scale Intelligent Microservices.md",97308],"56543a15":[()=>n.e(63402).then(n.bind(n,26786)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",26786],"567cd2fd":[()=>n.e(7201).then(n.bind(n,4423)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Image Explainers.md",4423],"56d039a1":[()=>Promise.all([n.e(40532),n.e(85729)]).then(n.bind(n,48482)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_cognitive.md",48482],"57e687e8":[()=>n.e(69159).then(n.bind(n,95823)),"@site/versioned_docs/version-0.11.3/Reference/Docker Setup.md",95823],"57ef17c5":[()=>n.e(95602).then(n.bind(n,82569)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",82569],"58486a98":[()=>Promise.all([n.e(40532),n.e(76572)]).then(n.bind(n,38697)),"@site/versioned_docs/version-1.0.4/Reference/Dotnet Setup.md",38697],"5887dc62":[()=>Promise.all([n.e(40532),n.e(77969)]).then(n.bind(n,89912)),"@site/versioned_docs/version-1.0.1/Reference/Dotnet Setup.md",89912],"589adaf2":[()=>Promise.all([n.e(40532),n.e(36572)]).then(n.bind(n,41965)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_causal.md",41965],"58b8f176":[()=>n.e(99374).then(n.bind(n,58838)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences.md",58838],"5962ef6e":[()=>Promise.all([n.e(40532),n.e(12630)]).then(n.bind(n,27505)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_core.md",27505],"59d9fecc":[()=>n.e(84908).then(n.bind(n,53217)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Overview.md",53217],"59e4d88e":[()=>n.e(2381).then(n.bind(n,60701)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/ONNX.md",60701],"59ef8022":[()=>n.e(48491).then(n.bind(n,23796)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis.md",23796],"5a3e1bca":[()=>n.e(94702).then(n.bind(n,83285)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",83285],"5b779334":[()=>n.e(16833).then(n.bind(n,88864)),"@site/docs/Reference/Quickstart - LightGBM in Dotnet.md",88864],"5bf420e9":[()=>Promise.all([n.e(40532),n.e(32855)]).then(n.bind(n,6805)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_core.md",6805],"5c0b440a":[()=>Promise.all([n.e(40532),n.e(50783)]).then(n.bind(n,2885)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_deep_learning.md",2885],"5c18deb5":[()=>n.e(33358).then(n.bind(n,24964)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Geospatial Services.md",24964],"5cfa133c":[()=>n.e(62079).then(n.bind(n,80697)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",80697],"5dbf4619":[()=>n.e(92867).then(n.bind(n,76669)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis.md",76669],"5e008bd1":[()=>n.e(6304).then(n.bind(n,20671)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",20671],"5e7c3303":[()=>n.e(38781).then(n.bind(n,72613)),"@site/docs/Explore Algorithms/Deep Learning/Distributed Training.md",72613],"5e9f5e1a":[()=>Promise.resolve().then(n.bind(n,36809)),"@generated/docusaurus.config",36809],"5eddefdd":[()=>n.e(30013).then(n.bind(n,61820)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",61820],"5ef28062":[()=>n.e(36746).then(n.bind(n,30356)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",30356],"5efd5006":[()=>n.e(37742).then(n.bind(n,41783)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",41783],"5f17911b":[()=>n.e(32599).then(n.bind(n,93084)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",93084],"603706f6":[()=>Promise.all([n.e(40532),n.e(15149)]).then(n.bind(n,43006)),"@site/versioned_docs/version-0.11.3/Reference/Dotnet Setup.md",43006],"604b2d50":[()=>Promise.all([n.e(40532),n.e(83790)]).then(n.bind(n,29364)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_vw.md",29364],"6094981e":[()=>n.e(84298).then(n.bind(n,78053)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",78053],"60a2189a":[()=>n.e(3953).then(n.bind(n,82469)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis.md",82469],"610d430a":[()=>n.e(40659).then(n.bind(n,33728)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Overview.md",33728],"627aefca":[()=>n.e(30301).then(n.t.bind(n,15745,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-pages/default/plugin-route-context-module-100.json",15745],"62983deb":[()=>n.e(76843).then(n.bind(n,19518)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Langchain.md",19518],"62ccfbde":[()=>n.e(82681).then(n.bind(n,70375)),"@site/versioned_docs/version-1.0.4/Get Started/Install SynapseML.md",70375],"63cd8b22":[()=>n.e(17973).then(n.bind(n,47831)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",47831],"644ff257":[()=>n.e(51910).then(n.bind(n,97780)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Text Explainers.md",97780],"645abc7d":[()=>n.e(31836).then(n.bind(n,32321)),"@site/docs/Reference/Docker Setup.md",32321],"66d63bfc":[()=>n.e(17542).then(n.t.bind(n,84370,19)),"~docs/default/version-1-0-3-metadata-prop-7b4.json",84370],"66fcb786":[()=>n.e(87597).then(n.bind(n,69554)),"@site/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",69554],"67e06a8b":[()=>n.e(86704).then(n.bind(n,11969)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",11969],"69bb4b11":[()=>Promise.all([n.e(40532),n.e(90288)]).then(n.bind(n,14233)),"@site/docs/Quick Examples/transformers/transformers_opencv.md",14233],"69dc6d0a":[()=>n.e(99161).then(n.bind(n,94325)),"@site/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md",94325],"6a07b55e":[()=>n.e(70406).then(n.bind(n,44243)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",44243],"6a26e359":[()=>n.e(33195).then(n.bind(n,35071)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",35071],"6a95f87e":[()=>n.e(73796).then(n.t.bind(n,56800,19)),"~blog/default/synapse-ml-blog-963.json",56800],"6a9ad78d":[()=>Promise.all([n.e(40532),n.e(51317)]).then(n.bind(n,5599)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_causal.md",5599],"6b40ec54":[()=>n.e(22712).then(n.bind(n,36338)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",36338],"6b9bdd6f":[()=>n.e(51105).then(n.bind(n,60038)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Geospatial Services.md",60038],"6bdbf6b8":[()=>n.e(98583).then(n.bind(n,351)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/LightGBM/Overview.md",351],"6c4c8509":[()=>n.e(42522).then(n.bind(n,25434)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",25434],"6c61f93b":[()=>Promise.all([n.e(40532),n.e(3202)]).then(n.bind(n,78968)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_causal.md",78968],"6c7ccb6d":[()=>n.e(12398).then(n.bind(n,18527)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",18527],"6c7d288d":[()=>n.e(36224).then(n.bind(n,70515)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",70515],"6c82e982":[()=>n.e(2070).then(n.bind(n,47636)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",47636],"6e159789":[()=>n.e(3611).then(n.bind(n,6930)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",6930],"6e2678b7":[()=>n.e(28465).then(n.bind(n,31750)),"@site/versioned_docs/version-1.0.3/Deploy Models/Quickstart - Deploying a Classifier.md",31750],"6e4e5cd9":[()=>n.e(21401).then(n.bind(n,39225)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",39225],"6e71bda3":[()=>n.e(28955).then(n.bind(n,63355)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/ONNX.md",63355],"6eddee4c":[()=>n.e(17295).then(n.bind(n,82586)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",82586],"6f479459":[()=>n.e(74963).then(n.bind(n,33261)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Image Explainers.md",33261],"6f8cd013":[()=>n.e(38078).then(n.bind(n,73529)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",73529],"6ff7775f":[()=>n.e(89461).then(n.bind(n,48404)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",48404],"707d2a35":[()=>n.e(6633).then(n.bind(n,38429)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine.md",38429],"70b306ba":[()=>n.e(38377).then(n.bind(n,61519)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/ONNX.md",61519],"70e8008e":[()=>n.e(88838).then(n.bind(n,12521)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Install.md",12521],"70fe7dd7":[()=>n.e(44178).then(n.bind(n,77849)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Overview.md",77849],71042800:[()=>n.e(59484).then(n.bind(n,30382)),"@site/versioned_docs/version-0.11.3/Reference/Quickstart - LightGBM in Dotnet.md",30382],"7275f11a":[()=>n.e(34940).then(n.bind(n,55516)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",55516],"7296bed0":[()=>n.e(98528).then(n.bind(n,97421)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",97421],"73c2022c":[()=>n.e(847).then(n.bind(n,32422)),"@site/docs/Get Started/Install SynapseML.md",32422],"73cf52e8":[()=>n.e(4026).then(n.bind(n,88585)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",88585],"74091a62":[()=>n.e(63196).then(n.bind(n,89086)),"@site/versioned_docs/version-1.0.2/Reference/R Setup.md",89086],"74659d33":[()=>n.e(66167).then(n.bind(n,82605)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine.md",82605],75406112:[()=>n.e(73060).then(n.bind(n,73375)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",73375],"75514f6a":[()=>n.e(12824).then(n.bind(n,95694)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor.md",95694],75801678:[()=>Promise.all([n.e(40532),n.e(6289)]).then(n.bind(n,62602)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_core.md",62602],"7649ec47":[()=>n.e(45370).then(n.bind(n,34329)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",34329],"767a7177":[()=>n.e(35040).then(n.bind(n,49352)),"@site/versioned_docs/version-0.11.4/Get Started/Set up Cognitive Services.md",49352],"76b137bf":[()=>n.e(55276).then(n.bind(n,70993)),"@site/versioned_docs/version-1.0.1/Reference/Contributor Guide.md",70993],"76bf568a":[()=>n.e(59823).then(n.t.bind(n,38484,19)),"~docs/default/version-1-0-4-metadata-prop-029.json",38484],"76d23901":[()=>n.e(16269).then(n.bind(n,46954)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",46954],"77c259dd":[()=>n.e(70224).then(n.bind(n,60878)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Overview.md",60878],"78aebd5d":[()=>n.e(91895).then(n.bind(n,38547)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",38547],"78cb5959":[()=>n.e(5909).then(n.bind(n,87957)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",87957],"79114dfe":[()=>n.e(41944).then(n.bind(n,92175)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers.md",92175],"792eae7e":[()=>Promise.all([n.e(40532),n.e(74275)]).then(n.bind(n,48117)),"@site/docs/Quick Examples/estimators/estimators_lightgbm.md",48117],"79d79762":[()=>Promise.all([n.e(40532),n.e(46845)]).then(n.bind(n,86931)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_cognitive.md",86931],"7a1d9beb":[()=>n.e(75088).then(n.bind(n,17021)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard.md",17021],"7aa49360":[()=>n.e(99147).then(n.bind(n,82379)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Overview.md",82379],"7aa7b8e0":[()=>Promise.all([n.e(40532),n.e(70344)]).then(n.bind(n,31036)),"@site/versioned_docs/version-0.11.4/Reference/Dotnet Setup.md",31036],"7ad384dc":[()=>n.e(45589).then(n.bind(n,60946)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/LightGBM/Overview.md",60946],"7aeb6f16":[()=>n.e(24046).then(n.bind(n,99342)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences.md",99342],"7b60d8e3":[()=>n.e(403).then(n.bind(n,47989)),"@site/versioned_docs/version-1.0.2/Reference/Contributor Guide.md",47989],"7bf16cc6":[()=>n.e(86248).then(n.bind(n,75144)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",75144],"7c007ec6":[()=>n.e(29283).then(n.bind(n,65302)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",65302],"7c2e1100":[()=>n.e(58861).then(n.bind(n,32961)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Langchain.md",32961],"80472ecc":[()=>n.e(91078).then(n.bind(n,43253)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis.md",43253],"814f3328":[()=>n.e(52535).then(n.t.bind(n,45641,19)),"~blog/default/blog-post-list-prop-default.json",45641],"8181e18a":[()=>n.e(69226).then(n.bind(n,15489)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Getting Started.md",15489],"82878c84":[()=>n.e(94916).then(n.bind(n,81962)),"@site/versioned_docs/version-0.11.3/Get Started/Create a Spark Cluster.md",81962],"82d7cc79":[()=>n.e(96816).then(n.bind(n,68560)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",68560],"83203dd1":[()=>n.e(89450).then(n.bind(n,8346)),"@site/versioned_docs/version-1.0.3/Get Started/Set up Cognitive Services.md",8346],"8328d740":[()=>n.e(91921).then(n.bind(n,18567)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers.md",18567],"83586bff":[()=>n.e(37001).then(n.bind(n,22760)),"@site/docs/Explore Algorithms/OpenCV/Image Transformations.md",22760],"8462374b":[()=>Promise.all([n.e(40532),n.e(66808)]).then(n.bind(n,14707)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_core.md",14707],"8590feaa":[()=>Promise.all([n.e(40532),n.e(91871)]).then(n.bind(n,53505)),"@site/docs/Quick Examples/transformers/transformers_vw.md",53505],"85aed2f9":[()=>n.e(91747).then(n.bind(n,31029)),"@site/versioned_docs/version-0.11.4/Reference/R Setup.md",31029],"866b91a1":[()=>n.e(83781).then(n.bind(n,23442)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Text Explainers.md",23442],"86ed1ff2":[()=>n.e(53090).then(n.bind(n,9814)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",9814],"8799ef55":[()=>n.e(45938).then(n.bind(n,13835)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",13835],"888e2229":[()=>n.e(67448).then(n.bind(n,9850)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",9850],"88b63415":[()=>n.e(360).then(n.bind(n,57933)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",57933],"88e44e99":[()=>Promise.all([n.e(40532),n.e(78817)]).then(n.bind(n,14859)),"@site/docs/Quick Examples/estimators/estimators_vw.md",14859],"88f8e3e1":[()=>n.e(22970).then(n.bind(n,69278)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenCV/Image Transformations.md",69278],"8a1bca9d":[()=>n.e(26871).then(n.bind(n,44359)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Overview.md",44359],"8a46fce0":[()=>n.e(92836).then(n.bind(n,59438)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",59438],"8af72580":[()=>n.e(22577).then(n.bind(n,84586)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Overview.md",84586],"8bff2444":[()=>n.e(19336).then(n.bind(n,17465)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Overview.md",17465],"8cb11643":[()=>n.e(95974).then(n.bind(n,8169)),"@site/blog/2019-10-02-MMLSpark empowering AI for Good with Mark Hamilton.md?truncated=true",8169],"8d990105":[()=>n.e(28456).then(n.bind(n,94078)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",94078],"8e32e44d":[()=>n.e(73419).then(n.bind(n,21554)),"@site/versioned_docs/version-0.11.3/Reference/Developer Setup.md",21554],"8eb438b4":[()=>n.e(85043).then(n.bind(n,34540)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",34540],"8fd0c721":[()=>n.e(80112).then(n.bind(n,99640)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Overview.md",99640],"8ff6772d":[()=>n.e(32777).then(n.bind(n,30213)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Tabular Explainers.md",30213],"900b935c":[()=>n.e(65942).then(n.bind(n,69180)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",69180],"905ce8b5":[()=>n.e(29535).then(n.bind(n,34527)),"@site/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",34527],90742975:[()=>n.e(36005).then(n.bind(n,41454)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis.md",41454],"90e4432b":[()=>n.e(15095).then(n.bind(n,40814)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",40814],"90f00051":[()=>n.e(33631).then(n.bind(n,2040)),"@site/versioned_docs/version-1.0.1/Get Started/Quickstart - Your First Models.md",2040],"91e9cb67":[()=>n.e(73443).then(n.bind(n,71766)),"@site/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",71766],"92ce4fd5":[()=>n.e(23838).then(n.bind(n,61997)),"@site/versioned_docs/version-1.0.2/Get Started/Set up Cognitive Services.md",61997],"935f2afb":[()=>n.e(80053).then(n.t.bind(n,1109,19)),"~docs/default/version-current-metadata-prop-751.json",1109],"93ba15e0":[()=>n.e(87452).then(n.bind(n,29850)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",29850],"943afd32":[()=>n.e(53050).then(n.bind(n,30322)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",30322],"94d743d6":[()=>n.e(94579).then(n.bind(n,12815)),"@site/docs/Deploy Models/Overview.md",12815],"9530a2cf":[()=>n.e(7509).then(n.bind(n,70325)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",70325],"95d9e891":[()=>n.e(63099).then(n.bind(n,6022)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",6022],"965ed185":[()=>Promise.all([n.e(40532),n.e(40852)]).then(n.bind(n,29009)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_cognitive.md",29009],"9688a5be":[()=>Promise.all([n.e(40532),n.e(51918)]).then(n.bind(n,52915)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_cognitive.md",52915],"96cf5ff0":[()=>n.e(46135).then(n.bind(n,35761)),"@site/docs/Explore Algorithms/OpenAI/OpenAI.md",35761],"97592aac":[()=>n.e(50653).then(n.bind(n,36490)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",36490],"9783ff24":[()=>Promise.all([n.e(40532),n.e(29006)]).then(n.bind(n,89448)),"@site/docs/Quick Examples/estimators/estimators_core.md",89448],"980b00ea":[()=>n.e(2901).then(n.bind(n,37673)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard.md",37673],"98258c5e":[()=>n.e(46559).then(n.bind(n,39475)),"@site/versioned_docs/version-1.0.2/Get Started/Install SynapseML.md",39475],"990f07de":[()=>n.e(58755).then(n.bind(n,27471)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",27471],"993c5022":[()=>n.e(42815).then(n.bind(n,16087)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/LightGBM/Overview.md",16087],"995576e9":[()=>n.e(43344).then(n.bind(n,61433)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",61433],"99e5ffa1":[()=>n.e(44504).then(n.bind(n,22327)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Getting Started.md",22327],"99f8fee5":[()=>n.e(67287).then(n.bind(n,1288)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",1288],"9b020c4a":[()=>n.e(52752).then(n.bind(n,69975)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",69975],"9c279ae7":[()=>n.e(50196).then(n.bind(n,91257)),"@site/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",91257],"9c5088a2":[()=>n.e(7925).then(n.bind(n,22299)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Autologging.md",22299],"9cb6fa7c":[()=>n.e(76515).then(n.bind(n,1458)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Overview.md",1458],"9d095dba":[()=>Promise.all([n.e(40532),n.e(57231)]).then(n.bind(n,53620)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_core.md",53620],"9da9112f":[()=>n.e(8006).then(n.bind(n,61363)),"@site/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",61363],"9e4087bc":[()=>n.e(53608).then(n.bind(n,63169)),"@theme/BlogArchivePage",63169],"9e5b788f":[()=>n.e(40412).then(n.bind(n,71393)),"@site/versioned_docs/version-1.0.2/Reference/Developer Setup.md",71393],"9fa7e8eb":[()=>n.e(31896).then(n.bind(n,34021)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Cyber ML.md",34021],a0434473:[()=>n.e(79328).then(n.bind(n,401)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Langchain.md",401],a07d09c1:[()=>n.e(27572).then(n.bind(n,82679)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",82679],a0d8a1a2:[()=>n.e(61778).then(n.bind(n,83269)),"@site/versioned_docs/version-1.0.2/Get Started/Create a Spark Cluster.md",83269],a1824f78:[()=>n.e(68070).then(n.bind(n,55261)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",55261],a19ce767:[()=>n.e(89463).then(n.bind(n,32152)),"@site/docs/Get Started/Create a Spark Cluster.md",32152],a2b5e5df:[()=>n.e(89220).then(n.bind(n,96797)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenCV/Image Transformations.md",96797],a34b09d4:[()=>n.e(95768).then(n.bind(n,25401)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",25401],a34c49d0:[()=>n.e(22007).then(n.bind(n,96633)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",96633],a3fd5a6d:[()=>Promise.all([n.e(40532),n.e(52730)]).then(n.bind(n,73778)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_opencv.md",73778],a406e231:[()=>n.e(37486).then(n.bind(n,30640)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",30640],a563fa93:[()=>n.e(8839).then(n.bind(n,82684)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",82684],a5ffe972:[()=>n.e(14341).then(n.bind(n,34581)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",34581],a608660b:[()=>n.e(67597).then(n.bind(n,90970)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",90970],a6191053:[()=>n.e(30927).then(n.bind(n,94736)),"@site/docs/Use with MLFlow/Overview.md",94736],a6aa9e1f:[()=>Promise.all([n.e(40532),n.e(24572),n.e(46048),n.e(93089)]).then(n.bind(n,93269)),"@theme/BlogListPage",93269],a81cf8b0:[()=>n.e(50040).then(n.bind(n,18943)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",18943],a8456d9f:[()=>n.e(53572).then(n.bind(n,61158)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",61158],a877f9e5:[()=>Promise.all([n.e(40532),n.e(72141)]).then(n.bind(n,69956)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_vw.md",69956],a9330763:[()=>n.e(15511).then(n.bind(n,36077)),"@site/versioned_docs/version-0.11.4/Deploy Models/Overview.md",36077],aa5548ab:[()=>n.e(22341).then(n.bind(n,73108)),"@site/versioned_docs/version-1.0.1/Deploy Models/Overview.md",73108],ab9a8589:[()=>n.e(80707).then(n.bind(n,61167)),"@site/blog/2019-06-01-MMLSpark Unifying Machine Learning Ecosystems at Massive Scales.md?truncated=true",61167],aba932dd:[()=>n.e(36698).then(n.bind(n,17798)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",17798],ac39f7dc:[()=>n.e(15404).then(n.bind(n,4200)),"@site/versioned_docs/version-1.0.3/Get Started/Quickstart - Your First Models.md",4200],ac581902:[()=>n.e(10978).then(n.bind(n,25386)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",25386],ad10988d:[()=>Promise.all([n.e(40532),n.e(1854)]).then(n.bind(n,36305)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_opencv.md",36305],ad139e3c:[()=>n.e(17538).then(n.bind(n,25612)),"@site/versioned_docs/version-0.11.4/Deploy Models/Quickstart - Deploying a Classifier.md",25612],ae89cd3b:[()=>n.e(39139).then(n.bind(n,46405)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",46405],af1e70b6:[()=>n.e(73602).then(n.bind(n,49816)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor.md",49816],affaa541:[()=>n.e(36673).then(n.bind(n,53727)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Train Regressor.md",53727],b066233f:[()=>n.e(28696).then(n.bind(n,37753)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",37753],b0e28723:[()=>n.e(36285).then(n.bind(n,10431)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",10431],b1706689:[()=>n.e(17655).then(n.bind(n,5358)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",5358],b28fe732:[()=>Promise.all([n.e(40532),n.e(87808)]).then(n.bind(n,99485)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_lightgbm.md",99485],b2be3901:[()=>n.e(19775).then(n.bind(n,83907)),"@site/docs/Get Started/Set up Cognitive Services.md",83907],b3272230:[()=>n.e(806).then(n.bind(n,90766)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",90766],b328a361:[()=>n.e(17782).then(n.bind(n,92522)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",92522],b33598ac:[()=>n.e(92059).then(n.bind(n,13052)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Image Explainers.md",13052],b35d004b:[()=>n.e(87384).then(n.bind(n,97178)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",97178],b35e9998:[()=>n.e(77390).then(n.bind(n,94975)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Autologging.md",94975],b38eec89:[()=>Promise.all([n.e(40532),n.e(87684)]).then(n.bind(n,97256)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_deep_learning.md",97256],b41f295c:[()=>n.e(3814).then(n.bind(n,17551)),"@site/versioned_docs/version-0.11.3/Reference/Contributor Guide.md",17551],b4898d44:[()=>n.e(46613).then(n.bind(n,30178)),"@site/versioned_docs/version-1.0.2/Overview.md",30178],b4a95996:[()=>n.e(51478).then(n.bind(n,1973)),"@site/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",1973],b4ae214b:[()=>n.e(36773).then(n.bind(n,80609)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",80609],b5375b6f:[()=>n.e(61428).then(n.bind(n,7902)),"@site/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",7902],b576c645:[()=>n.e(80322).then(n.bind(n,39461)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers.md",39461],b66ab6b5:[()=>n.e(688).then(n.bind(n,37483)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",37483],b66e2100:[()=>n.e(41719).then(n.bind(n,16930)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Distributed Training.md",16930],b7226801:[()=>n.e(36585).then(n.bind(n,24646)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",24646],b72abe57:[()=>n.e(2195).then(n.bind(n,83847)),"@site/blog/2020-12-01-Large-Scale Intelligent Microservices.md?truncated=true",83847],b739c60a:[()=>n.e(30023).then(n.bind(n,49580)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",49580],b75118f0:[()=>n.e(61442).then(n.bind(n,40674)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",40674],b7787e72:[()=>n.e(15253).then(n.bind(n,23339)),"@site/versioned_docs/version-1.0.1/Reference/R Setup.md",23339],b7802ae1:[()=>n.e(38859).then(n.bind(n,49665)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",49665],b7c6295c:[()=>n.e(64233).then(n.bind(n,71955)),"@site/versioned_docs/version-1.0.1/Reference/Developer Setup.md",71955],b80b493a:[()=>n.e(86856).then(n.bind(n,92934)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",92934],b83021be:[()=>n.e(587).then(n.bind(n,6283)),"@site/docs/Explore Algorithms/Causal Inference/Overview.md",6283],b88f1f9b:[()=>n.e(53338).then(n.bind(n,92235)),"@site/versioned_docs/version-1.0.4/Reference/Developer Setup.md",92235],b8963dc0:[()=>n.e(5287).then(n.bind(n,65710)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",65710],b92ffdc6:[()=>n.e(80874).then(n.bind(n,29864)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",29864],b940e0b5:[()=>n.e(58738).then(n.bind(n,7662)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",7662],b962e007:[()=>n.e(24594).then(n.bind(n,46495)),"@site/blog/2019-10-02-MMLSpark empowering AI for Good with Mark Hamilton.md",46495],b9f125cc:[()=>n.e(29806).then(n.bind(n,43252)),"@site/versioned_docs/version-0.11.4/Overview.md",43252],ba9c0924:[()=>n.e(67218).then(n.bind(n,66174)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",66174],bb7a48da:[()=>Promise.all([n.e(40532),n.e(39242)]).then(n.bind(n,52528)),"@site/docs/Quick Examples/transformers/transformers_core.md",52528],bc4bf151:[()=>n.e(91755).then(n.bind(n,7297)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",7297],bc6a9944:[()=>n.e(98462).then(n.bind(n,38334)),"@site/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",38334],bc85b804:[()=>Promise.all([n.e(40532),n.e(45503)]).then(n.bind(n,20408)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_lightgbm.md",20408],bd1bcb2c:[()=>n.e(95584).then(n.bind(n,66193)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",66193],be95fd8f:[()=>n.e(57887).then(n.bind(n,8801)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",8801],bece3771:[()=>n.e(55538).then(n.bind(n,57450)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",57450],bef0d2d3:[()=>n.e(16286).then(n.bind(n,3766)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Cyber ML.md",3766],bf002efe:[()=>n.e(53017).then(n.bind(n,1347)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",1347],bf04857e:[()=>n.e(38846).then(n.bind(n,76282)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",76282],bf8a5c8a:[()=>n.e(63002).then(n.bind(n,42365)),"@site/versioned_docs/version-1.0.4/Overview.md",42365],c0391845:[()=>n.e(23234).then(n.bind(n,36422)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",36422],c065fcb9:[()=>n.e(4800).then(n.bind(n,66225)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",66225],c14fdd92:[()=>n.e(19042).then(n.bind(n,72368)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",72368],c23b5e26:[()=>n.e(75658).then(n.bind(n,75410)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",75410],c2492d56:[()=>n.e(84202).then(n.bind(n,35967)),"@site/versioned_docs/version-0.11.4/Get Started/Create a Spark Cluster.md",35967],c2e40c34:[()=>n.e(85122).then(n.bind(n,99967)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",99967],c2e57a1b:[()=>n.e(26685).then(n.bind(n,39133)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Text Explainers.md",39133],c34b1e36:[()=>n.e(83251).then(n.bind(n,54824)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",54824],c38c658c:[()=>n.e(62565).then(n.bind(n,91960)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",91960],c39e3602:[()=>Promise.all([n.e(40532),n.e(46807)]).then(n.bind(n,34717)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_opencv.md",34717],c3b1d949:[()=>n.e(35056).then(n.bind(n,84577)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",84577],c3c516ff:[()=>n.e(65752).then(n.bind(n,68928)),"@site/blog/overview.md?truncated=true",68928],c3dac62d:[()=>n.e(35537).then(n.bind(n,8045)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",8045],c40984d1:[()=>Promise.all([n.e(40532),n.e(87320)]).then(n.bind(n,64356)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_causal.md",64356],c413b43a:[()=>n.e(89767).then(n.bind(n,91730)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",91730],c4d09a44:[()=>n.e(52406).then(n.bind(n,48327)),"@site/versioned_docs/version-0.11.3/Get Started/Quickstart - Your First Models.md",48327],c4d68582:[()=>n.e(429).then(n.bind(n,2351)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",2351],c4f5d8e4:[()=>Promise.all([n.e(40532),n.e(87780),n.e(64195)]).then(n.bind(n,35567)),"@site/src/pages/index.js",35567],c5775233:[()=>n.e(47454).then(n.bind(n,21794)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/OpenAI.md",21794],c5c95e45:[()=>n.e(24422).then(n.bind(n,73290)),"@site/docs/Explore Algorithms/Responsible AI/Image Explainers.md",73290],c73cfc52:[()=>n.e(18825).then(n.bind(n,75084)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenCV/Image Transformations.md",75084],c764f2c0:[()=>n.e(97030).then(n.bind(n,37506)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Langchain.md",37506],c76e8cac:[()=>n.e(11298).then(n.bind(n,22344)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",22344],c82134d8:[()=>n.e(29540).then(n.bind(n,50500)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",50500],c90c942b:[()=>Promise.all([n.e(40532),n.e(91824)]).then(n.bind(n,21475)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_core.md",21475],c95882d3:[()=>n.e(31153).then(n.bind(n,19210)),"@site/docs/Use with MLFlow/Autologging.md",19210],c991f47b:[()=>n.e(68549).then(n.bind(n,6034)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor.md",6034],ca0a1bad:[()=>n.e(55305).then(n.bind(n,22468)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",22468],cb5f1a06:[()=>n.e(54528).then(n.bind(n,15346)),"@site/blog/overview.md",15346],cb7c2a83:[()=>n.e(21933).then(n.bind(n,24630)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",24630],cbc3190d:[()=>n.e(75488).then(n.bind(n,85418)),"@site/docs/Overview.md",85418],cc52e3cd:[()=>n.e(91214).then(n.bind(n,70650)),"@site/versioned_docs/version-1.0.4/Get Started/Create a Spark Cluster.md",70650],ccc49370:[()=>Promise.all([n.e(40532),n.e(24572),n.e(46048),n.e(46103)]).then(n.bind(n,65203)),"@theme/BlogPostPage",65203],cd8e97a5:[()=>n.e(44422).then(n.t.bind(n,53485,19)),"~docs/default/version-1-0-1-metadata-prop-e87.json",53485],ce40236b:[()=>n.e(27497).then(n.bind(n,18347)),"@site/versioned_docs/version-1.0.4/Get Started/Set up Cognitive Services.md",18347],ce7af6ae:[()=>n.e(55864).then(n.bind(n,7156)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Explanation Dashboard.md",7156],cee4c3b9:[()=>n.e(25844).then(n.bind(n,30877)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",30877],ceeda143:[()=>n.e(18443).then(n.bind(n,31146)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",31146],cf57716c:[()=>n.e(5e3).then(n.bind(n,74832)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",74832],cf666d49:[()=>Promise.all([n.e(40532),n.e(25449)]).then(n.bind(n,34621)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_causal.md",34621],cff2e56e:[()=>n.e(56373).then(n.bind(n,65611)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",65611],d05981ec:[()=>n.e(12124).then(n.bind(n,94929)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Geospatial Services.md",94929],d0c5e4fa:[()=>Promise.all([n.e(40532),n.e(95717)]).then(n.bind(n,77718)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_deep_learning.md",77718],d0cb74df:[()=>n.e(94597).then(n.bind(n,82689)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",82689],d1246f49:[()=>Promise.all([n.e(40532),n.e(69966)]).then(n.bind(n,16082)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_lightgbm.md",16082],d13e532e:[()=>n.e(8146).then(n.bind(n,90398)),"@site/docs/Explore Algorithms/Responsible AI/Text Explainers.md",90398],d1428557:[()=>n.e(3765).then(n.bind(n,25792)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/OpenAI.md",25792],d175df5e:[()=>n.e(92805).then(n.t.bind(n,7812,19)),"~blog/default/synapse-ml-blog-archive-c02.json",7812],d252041b:[()=>n.e(76300).then(n.bind(n,29685)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Distributed Training.md",29685],d25cde70:[()=>n.e(81102).then(n.bind(n,96397)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",96397],d3093636:[()=>n.e(77491).then(n.bind(n,94648)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",94648],d355047f:[()=>n.e(64713).then(n.bind(n,77696)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Overview.md",77696],d36f3429:[()=>n.e(91387).then(n.bind(n,87735)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Overview.md",87735],d39aa6d3:[()=>n.e(57930).then(n.bind(n,91123)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Install.md",91123],d3c9b0d7:[()=>n.e(98050).then(n.bind(n,9151)),"@site/versioned_docs/version-0.11.3/Get Started/Install SynapseML.md",9151],d41234a3:[()=>n.e(44177).then(n.bind(n,2193)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",2193],d535b358:[()=>n.e(88984).then(n.bind(n,62051)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",62051],d57515f7:[()=>n.e(37638).then(n.bind(n,18898)),"@site/versioned_docs/version-1.0.3/Reference/R Setup.md",18898],d620a661:[()=>n.e(28746).then(n.bind(n,35215)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",35215],d707d2ed:[()=>n.e(83669).then(n.bind(n,35018)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",35018],d7f58694:[()=>n.e(94553).then(n.bind(n,32484)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",32484],d81d1137:[()=>n.e(44490).then(n.bind(n,81287)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",81287],d84904f9:[()=>n.e(18563).then(n.bind(n,67607)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers.md",67607],d8b3ac6a:[()=>n.e(94330).then(n.bind(n,7627)),"@site/docs/Explore Algorithms/Other Algorithms/Cyber ML.md",7627],d8d6ba90:[()=>n.e(51842).then(n.t.bind(n,83769,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-docs/default/plugin-route-context-module-100.json",83769],d95a70ea:[()=>n.e(62721).then(n.bind(n,32398)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenCV/Image Transformations.md",32398],d9837698:[()=>n.e(57168).then(n.bind(n,80927)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/LightGBM/Overview.md",80927],d989c707:[()=>n.e(58462).then(n.bind(n,66657)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Overview.md",66657],d9e5e528:[()=>Promise.all([n.e(40532),n.e(38057)]).then(n.bind(n,58319)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_cognitive.md",58319],da508cef:[()=>n.e(33064).then(n.bind(n,16685)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenCV/Image Transformations.md",16685],dac82c00:[()=>Promise.all([n.e(40532),n.e(28559)]).then(n.bind(n,21888)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_cognitive.md",21888],db170a23:[()=>n.e(67706).then(n.bind(n,15710)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",15710],dc1e40d7:[()=>n.e(27991).then(n.t.bind(n,5244,19)),"~docs/default/version-1-0-2-metadata-prop-290.json",5244],dc7f7ac1:[()=>n.e(92018).then(n.bind(n,82058)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",82058],dd29718b:[()=>n.e(15579).then(n.bind(n,6692)),"@site/versioned_docs/version-0.11.4/Reference/Developer Setup.md",6692],dd3f9ada:[()=>n.e(17820).then(n.bind(n,66851)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",66851],ddacd194:[()=>n.e(62465).then(n.bind(n,49718)),"@site/versioned_docs/version-1.0.3/Deploy Models/Overview.md",49718],de3bb7c3:[()=>n.e(42970).then(n.bind(n,13725)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",13725],dea122e2:[()=>n.e(99781).then(n.bind(n,77541)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",77541],deef0af8:[()=>n.e(89407).then(n.bind(n,41983)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",41983],df01388f:[()=>Promise.all([n.e(40532),n.e(36828)]).then(n.bind(n,36579)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_vw.md",36579],dfb6dfe9:[()=>n.e(59076).then(n.bind(n,48770)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",48770],e0077969:[()=>n.e(97297).then(n.bind(n,43415)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Distributed Training.md",43415],e03f5a54:[()=>n.e(82569).then(n.bind(n,44581)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Cyber ML.md",44581],e1269457:[()=>n.e(80598).then(n.bind(n,10036)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",10036],e12b4691:[()=>n.e(10912).then(n.bind(n,19252)),"@site/versioned_docs/version-1.0.2/Deploy Models/Overview.md",19252],e1b2b5f3:[()=>n.e(81340).then(n.bind(n,60269)),"@site/blog/2018-04-01-Flexible and Scalable Deep Learning with MMLSpark.md",60269],e1ed1e14:[()=>n.e(82880).then(n.bind(n,3721)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenCV/Image Transformations.md",3721],e21a3367:[()=>n.e(42638).then(n.bind(n,8940)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Geospatial Services.md",8940],e24ee294:[()=>Promise.all([n.e(40532),n.e(8040)]).then(n.bind(n,73198)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_core.md",73198],e2689dfc:[()=>n.e(57258).then(n.bind(n,10290)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",10290],e2ce57ed:[()=>n.e(91210).then(n.bind(n,50102)),"@site/blog/2019-06-01-MMLSpark Unifying Machine Learning Ecosystems at Massive Scales.md",50102],e3089280:[()=>n.e(42283).then(n.bind(n,88617)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Langchain.md",88617],e316de39:[()=>n.e(22011).then(n.bind(n,74053)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",74053],e402e26d:[()=>Promise.all([n.e(40532),n.e(94623)]).then(n.bind(n,2570)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_cognitive.md",2570],e464d112:[()=>n.e(94508).then(n.bind(n,50791)),"@site/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",50791],e4ba4487:[()=>n.e(81787).then(n.bind(n,43982)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",43982],e4e95844:[()=>n.e(2797).then(n.bind(n,16230)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",16230],e579bad6:[()=>n.e(24627).then(n.bind(n,18579)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",18579],e5c4ef01:[()=>n.e(96673).then(n.bind(n,95705)),"@site/versioned_docs/version-0.11.4/Reference/Docker Setup.md",95705],e5fe80e3:[()=>n.e(66419).then(n.bind(n,60673)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",60673],e7250a93:[()=>n.e(75586).then(n.bind(n,41630)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",41630],e7bd83ca:[()=>Promise.all([n.e(40532),n.e(8168)]).then(n.bind(n,51453)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_vw.md",51453],e84755e1:[()=>n.e(68283).then(n.bind(n,69821)),"@site/versioned_docs/version-1.0.4/Get Started/Quickstart - Your First Models.md",69821],e863531f:[()=>n.e(66981).then(n.bind(n,51460)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",51460],e887b273:[()=>n.e(34654).then(n.bind(n,65011)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",65011],e8b9c2c7:[()=>n.e(84979).then(n.bind(n,28781)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",28781],e8d66fc1:[()=>n.e(271).then(n.bind(n,33942)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",33942],e9aab890:[()=>n.e(51732).then(n.bind(n,48613)),"@site/versioned_docs/version-0.11.4/Reference/Contributor Guide.md",48613],e9b75d67:[()=>n.e(95064).then(n.bind(n,28314)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/OpenAI.md",28314],ea2d2bcc:[()=>n.e(4204).then(n.bind(n,63800)),"@site/docs/Explore Algorithms/Deep Learning/ONNX.md",63800],eb0ef952:[()=>n.e(98197).then(n.bind(n,89595)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",89595],ebf648dd:[()=>Promise.all([n.e(40532),n.e(15566)]).then(n.bind(n,33154)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_vw.md",33154],ed316aed:[()=>n.e(67870).then(n.bind(n,71351)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",71351],ed40d5f1:[()=>n.e(6712).then(n.bind(n,21782)),"@site/versioned_docs/version-1.0.3/Reference/Developer Setup.md",21782],ed5c6c48:[()=>n.e(52787).then(n.bind(n,99421)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",99421],ed6d544d:[()=>n.e(63448).then(n.bind(n,79176)),"@site/docs/Explore Algorithms/OpenAI/Langchain.md",79176],ee6cea3b:[()=>n.e(80829).then(n.bind(n,56433)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Geospatial Services.md",56433],ee83122d:[()=>n.e(85648).then(n.bind(n,25)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",25],eed630e0:[()=>n.e(49101).then(n.bind(n,88069)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",88069],ef6ec597:[()=>n.e(13482).then(n.bind(n,88889)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Distributed Training.md",88889],ef77ef39:[()=>n.e(37392).then(n.bind(n,75405)),"@site/versioned_docs/version-1.0.2/Reference/Quickstart - LightGBM in Dotnet.md",75405],efbb59fb:[()=>n.e(99378).then(n.bind(n,47415)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",47415],efbc69e1:[()=>n.e(49196).then(n.bind(n,87949)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Overview.md",87949],efc1978d:[()=>n.e(79079).then(n.bind(n,25012)),"@site/versioned_docs/version-1.0.2/Deploy Models/Quickstart - Deploying a Classifier.md",25012],f027bb15:[()=>n.e(36916).then(n.bind(n,53307)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Image Explainers.md",53307],f0649d54:[()=>n.e(60880).then(n.bind(n,82604)),"@site/versioned_docs/version-1.0.3/Reference/Quickstart - LightGBM in Dotnet.md",82604],f14633e7:[()=>Promise.all([n.e(40532),n.e(84207)]).then(n.bind(n,61966)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_lightgbm.md",61966],f1b2dc7f:[()=>n.e(28634).then(n.bind(n,56309)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",56309],f21c13c4:[()=>n.e(23807).then(n.bind(n,42334)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier.md",42334],f228fdc5:[()=>n.e(82643).then(n.bind(n,66874)),"@site/versioned_docs/version-1.0.3/Get Started/Create a Spark Cluster.md",66874],f264933a:[()=>n.e(1283).then(n.bind(n,99121)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",99121],f322d2da:[()=>n.e(60445).then(n.bind(n,34609)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",34609],f38a0cbd:[()=>Promise.all([n.e(40532),n.e(61829)]).then(n.bind(n,66349)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_core.md",66349],f3e986ee:[()=>n.e(15788).then(n.bind(n,28310)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Getting Started.md",28310],f537d1e9:[()=>n.e(54890).then(n.bind(n,5773)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",5773],f5b4e517:[()=>n.e(98546).then(n.bind(n,23715)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",23715],f7580424:[()=>n.e(82158).then(n.bind(n,74640)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Autologging.md",74640],f7a32432:[()=>Promise.all([n.e(40532),n.e(84954)]).then(n.bind(n,19193)),"@site/docs/Quick Examples/transformers/transformers_deep_learning.md",19193],f7bfff73:[()=>Promise.all([n.e(40532),n.e(94603)]).then(n.bind(n,81363)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_opencv.md",81363],f8597cb0:[()=>n.e(82744).then(n.bind(n,25907)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",25907],f85db098:[()=>Promise.all([n.e(40532),n.e(95049)]).then(n.bind(n,46420)),"@site/docs/Reference/Dotnet Setup.md",46420],f8aab3ca:[()=>n.e(84880).then(n.bind(n,29469)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",29469],fa2e2688:[()=>n.e(64927).then(n.bind(n,78030)),"@site/docs/Explore Algorithms/AI Services/Overview.md",78030],fa889bee:[()=>n.e(81878).then(n.bind(n,91966)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",91966],faca745a:[()=>Promise.all([n.e(40532),n.e(67664)]).then(n.bind(n,44565)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_core.md",44565],fba72e87:[()=>n.e(11542).then(n.bind(n,20970)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",20970],fbb011bb:[()=>n.e(83378).then(n.bind(n,20693)),"@site/docs/Use with MLFlow/Install.md",20693],fc338b02:[()=>n.e(5378).then(n.t.bind(n,7085,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-theme-search-algolia/default/plugin-route-context-module-100.json",7085],fc47fc53:[()=>n.e(99601).then(n.bind(n,44088)),"@site/versioned_docs/version-1.0.1/Get Started/Install SynapseML.md",44088],fc598b5d:[()=>n.e(50002).then(n.bind(n,89076)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Overview.md",89076],fd3b6ea5:[()=>n.e(31279).then(n.bind(n,7922)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",7922],fd4bd09d:[()=>n.e(1286).then(n.bind(n,70781)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier.md",70781],fd748117:[()=>n.e(68809).then(n.bind(n,1359)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",1359],fe2c893b:[()=>n.e(50297).then(n.bind(n,18859)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier.md",18859],fe5d19cb:[()=>n.e(97065).then(n.bind(n,4460)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",4460],fe6131e8:[()=>n.e(44539).then(n.bind(n,17054)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",17054],fece3c24:[()=>Promise.all([n.e(40532),n.e(40367)]).then(n.bind(n,88387)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_core.md",88387],ff428354:[()=>n.e(51078).then(n.bind(n,92498)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",92498],ffb7f88a:[()=>n.e(85433).then(n.t.bind(n,81421,19)),"~docs/default/version-0-11-3-metadata-prop-9e0.json",81421]};function l(e){let{error:t,retry:n,pastDelay:r}=e;return t?s.createElement("div",{style:{textAlign:"center",color:"#fff",backgroundColor:"#fa383e",borderColor:"#fa383e",borderStyle:"solid",borderRadius:"0.25rem",borderWidth:"1px",boxSizing:"border-box",display:"block",padding:"1rem",flex:"0 0 50%",marginLeft:"25%",marginRight:"25%",marginTop:"5rem",maxWidth:"50%",width:"100%"}},s.createElement("p",null,String(t)),s.createElement("div",null,s.createElement("button",{type:"button",onClick:n},"Retry"))):r?s.createElement("div",{style:{display:"flex",justifyContent:"center",alignItems:"center",height:"100vh"}},s.createElement("svg",{id:"loader",style:{width:128,height:110,position:"absolute",top:"calc(100vh - 64%)"},viewBox:"0 0 45 45",xmlns:"http://www.w3.org/2000/svg",stroke:"#61dafb"},s.createElement("g",{fill:"none",fillRule:"evenodd",transform:"translate(1 1)",strokeWidth:"2"},s.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},s.createElement("animate",{attributeName:"r",begin:"1.5s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-opacity",begin:"1.5s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-width",begin:"1.5s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),s.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},s.createElement("animate",{attributeName:"r",begin:"3s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-opacity",begin:"3s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-width",begin:"3s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),s.createElement("circle",{cx:"22",cy:"22",r:"8"},s.createElement("animate",{attributeName:"r",begin:"0s",dur:"1.5s",values:"6;1;2;3;4;5;6",calcMode:"linear",repeatCount:"indefinite"}))))):null}var d=n(99670),p=n(30226);function u(e,t){if("*"===e)return a()({loading:l,loader:()=>Promise.all([n.e(40532),n.e(53140)]).then(n.bind(n,53140)),modules:["@theme/NotFound"],webpack:()=>[53140],render(e,t){const n=e.default;return s.createElement(p.z,{value:{plugin:{name:"native",id:"default"}}},s.createElement(n,t))}});const o=i[e+"-"+t],u={},m=[],h=[],g=(0,d.Z)(o);return Object.entries(g).forEach((e=>{let[t,n]=e;const s=c[n];s&&(u[t]=s[0],m.push(s[1]),h.push(s[2]))})),a().Map({loading:l,loader:u,modules:m,webpack:()=>h,render(t,n){const a=JSON.parse(JSON.stringify(o));Object.entries(t).forEach((t=>{let[n,s]=t;const r=s.default;if(!r)throw new Error("The page component at "+e+" doesn't have a default export. This makes it impossible to render anything. Consider default-exporting a React component.");"object"!=typeof r&&"function"!=typeof r||Object.keys(s).filter((e=>"default"!==e)).forEach((e=>{r[e]=s[e]}));let o=a;const i=n.split(".");i.slice(0,-1).forEach((e=>{o=o[e]})),o[i[i.length-1]]=r}));const i=a.__comp;delete a.__comp;const c=a.__context;return delete a.__context,s.createElement(p.z,{value:c},s.createElement(i,(0,r.Z)({},a,n)))}})}const m=[{path:"/SynapseML/blog/",component:u("/SynapseML/blog/","32c"),exact:!0},{path:"/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/",component:u("/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/","ecd"),exact:!0},{path:"/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/",component:u("/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/","870"),exact:!0},{path:"/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/",component:u("/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/","83d"),exact:!0},{path:"/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/",component:u("/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/","347"),exact:!0},{path:"/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/",component:u("/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/","546"),exact:!0},{path:"/SynapseML/blog/archive/",component:u("/SynapseML/blog/archive/","140"),exact:!0},{path:"/SynapseML/blog/overview/",component:u("/SynapseML/blog/overview/","32f"),exact:!0},{path:"/SynapseML/search/",component:u("/SynapseML/search/","a70"),exact:!0},{path:"/SynapseML/videos/",component:u("/SynapseML/videos/","9fc"),exact:!0},{path:"/SynapseML/docs/0.11.3/",component:u("/SynapseML/docs/0.11.3/","649"),routes:[{path:"/SynapseML/docs/0.11.3/Deploy Models/Overview/",component:u("/SynapseML/docs/0.11.3/Deploy Models/Overview/","107"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/","7ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","428"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/","20d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","dd0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/","8b1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","7f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/","9df"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/","398"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","475"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","746"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","00a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","be4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","4ed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/","f94"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","af8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","c3d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","1f2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/","d3c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","b90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","583"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/","c90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/","59d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/","4d0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","b98"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","52d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","018"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","086"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","2e6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","cd5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/","7d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","c77"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/","f47"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/","ffd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","c37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","3dc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/","5b7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/","e89"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","22c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","6da"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","4f5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/","b53"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/","e82"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","eb4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/","aa0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/","547"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/","ca9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","1c8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","850"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","103"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","22a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/","4a3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/","51c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","79d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","dbd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/","f6c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","899"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","de1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","6e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/","604"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Install SynapseML/",component:u("/SynapseML/docs/0.11.3/Get Started/Install SynapseML/","53d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/","bcf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/","e8a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Overview/",component:u("/SynapseML/docs/0.11.3/Overview/","201"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/","f33"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/","d03"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/","0fc"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/","113"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/","bcf"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/","3f4"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/","abc"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/","b7d"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/","a66"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/","b96"),exact:!0},{path:"/SynapseML/docs/0.11.3/Reference/Contributor Guide/",component:u("/SynapseML/docs/0.11.3/Reference/Contributor Guide/","06f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Developer Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Developer Setup/","cfc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Docker Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Docker Setup/","419"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Dotnet Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Dotnet Setup/","46d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/","9b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/R Setup/",component:u("/SynapseML/docs/0.11.3/Reference/R Setup/","1a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/","6a1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Install/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Install/","aa4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Overview/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Overview/","a43"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/0.11.4/",component:u("/SynapseML/docs/0.11.4/","72b"),routes:[{path:"/SynapseML/docs/0.11.4/Deploy Models/Overview/",component:u("/SynapseML/docs/0.11.4/Deploy Models/Overview/","1a4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/","ceb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","f98"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/","9e7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","2c3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/","be8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","1d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/","31e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/","8c4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","4f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","d9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","e8f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","8e2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/","78b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","756"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","264"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","c8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/","247"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","c00"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","c42"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/","53f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/","2d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/","169"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","b68"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","ff8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","016"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","6a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","b11"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/","e22"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","934"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/","fea"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/","cfe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","78f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","163"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/","29e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/","747"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","faf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","566"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","491"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/","7a4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/","75f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","2b0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/","f95"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/","66b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/","e63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","783"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","e01"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","3b8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","80b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/","18e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/","128"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","02f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","6d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/","b97"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","6ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","c86"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","964"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/","cb4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Install SynapseML/",component:u("/SynapseML/docs/0.11.4/Get Started/Install SynapseML/","34f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/","6fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/","528"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Overview/",component:u("/SynapseML/docs/0.11.4/Overview/","83a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/","32e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/","269"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/","11e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/","c94"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/","bf0"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/","728"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/","12e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/","1cc"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/","8fb"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/","828"),exact:!0},{path:"/SynapseML/docs/0.11.4/Reference/Contributor Guide/",component:u("/SynapseML/docs/0.11.4/Reference/Contributor Guide/","881"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Developer Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Developer Setup/","171"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Docker Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Docker Setup/","f85"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Dotnet Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Dotnet Setup/","f28"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/","553"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/R Setup/",component:u("/SynapseML/docs/0.11.4/Reference/R Setup/","fd3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/","3cb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Install/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Install/","360"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Overview/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Overview/","4e1"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.1/",component:u("/SynapseML/docs/1.0.1/","c9e"),routes:[{path:"/SynapseML/docs/1.0.1/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.1/Deploy Models/Overview/","7aa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/","c4a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","d96"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/","a96"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","4e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/","a84"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","e94"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/","be5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","76f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","5fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","417"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","cb0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","831"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","ce6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/","a37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","0db"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","d46"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","b74"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/","109"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","de7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","250"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/","68a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/","920"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/","4a5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","a53"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9d3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","da8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","b25"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","18d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","f90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/","15c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","796"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/","045"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/","5e3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","6f5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","457"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/","f73"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/","0f7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","fa5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","201"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","bd7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/","979"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/","810"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","1e9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/","6f0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/","55c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/","e7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","df0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","94a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","41d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","d72"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/","895"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/","b41"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","c46"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","728"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/","21f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","a43"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","7db"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","9d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/","697"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.1/Get Started/Install SynapseML/","349"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/","d39"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/","2fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Overview/",component:u("/SynapseML/docs/1.0.1/Overview/","64e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/","8d4"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/","d28"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/","1b9"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/","b23"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/","86c"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/","03e"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/","0db"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/","f4e"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/","945"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/","369"),exact:!0},{path:"/SynapseML/docs/1.0.1/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.1/Reference/Contributor Guide/","5ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Developer Setup/","971"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Docker Setup/","e6e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Dotnet Setup/","179"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/","049"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/R Setup/",component:u("/SynapseML/docs/1.0.1/Reference/R Setup/","392"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/","23e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Install/","0df"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Overview/","d32"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.2/",component:u("/SynapseML/docs/1.0.2/","df4"),routes:[{path:"/SynapseML/docs/1.0.2/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.2/Deploy Models/Overview/","a7c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/","7f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","185"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/","fa7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","1b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/","78e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","400"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/","5da"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","286"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","c7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","1d5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","d5d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","46e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","4ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/","8e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","405"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","2d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","764"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/","f3b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","8c6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","40b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/","a43"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/","001"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/","a2e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","613"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","5c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","893"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","bb7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","ccf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","a9c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/","108"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","a01"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/","f2d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/","a4a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","a0a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","5bb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/","6ae"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/","4c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","056"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","a63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","1e8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/","6cd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/","318"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","a37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/","eed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/","25a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/","810"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","5c9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","b99"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","3dd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","1bd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/","0b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/","250"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","bd7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","b2f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/","5a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","4ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","d16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","157"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/","3f8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.2/Get Started/Install SynapseML/","01e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/","0aa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/","095"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Overview/",component:u("/SynapseML/docs/1.0.2/Overview/","c78"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/","348"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/","386"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/","585"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/","fa7"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/","b73"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/","762"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/","7ec"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/","bad"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/","183"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/","029"),exact:!0},{path:"/SynapseML/docs/1.0.2/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.2/Reference/Contributor Guide/","8f1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Developer Setup/","5bf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Docker Setup/","294"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Dotnet Setup/","b72"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/","595"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/R Setup/",component:u("/SynapseML/docs/1.0.2/Reference/R Setup/","a22"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/","e9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Install/","7d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Overview/","e3c"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.3/",component:u("/SynapseML/docs/1.0.3/","60e"),routes:[{path:"/SynapseML/docs/1.0.3/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.3/Deploy Models/Overview/","1cf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/","695"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","0c3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/","e88"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","6d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/","0d9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","5bc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/","3ee"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","7b8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","5f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","85d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","6e8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","e1a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","54d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/","562"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","734"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","410"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","823"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/","1ea"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","96b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","432"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/","f57"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/","141"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/","005"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","2d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","abe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","fb6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","1fe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","561"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","158"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/","e1b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","99d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/","1f6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/","b8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","3b4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","146"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/","57d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/","e16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","3d8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","30f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","e7e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/","8c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/","1f6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","ea9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/","211"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/","18d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/","d3d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","025"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","878"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","326"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","a63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/","8ed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/","893"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","747"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","484"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/","60f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","c16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","35d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","f74"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/","ed3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.3/Get Started/Install SynapseML/","df3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/","450"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/","dff"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Overview/",component:u("/SynapseML/docs/1.0.3/Overview/","094"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/","505"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/","6ff"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/","104"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/","08b"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/","74d"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/","baf"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/","dcc"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/","80b"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/","4af"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/","9cd"),exact:!0},{path:"/SynapseML/docs/1.0.3/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.3/Reference/Contributor Guide/","676"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Developer Setup/","904"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Docker Setup/","963"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Dotnet Setup/","c7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/","9f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/R Setup/",component:u("/SynapseML/docs/1.0.3/Reference/R Setup/","3a0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/","00b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Install/","5a1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Overview/","234"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/next/",component:u("/SynapseML/docs/next/","b5f"),routes:[{path:"/SynapseML/docs/next/Deploy Models/Overview/",component:u("/SynapseML/docs/next/Deploy Models/Overview/","29d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/","eeb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","12a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/","00a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","e9b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/","0d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","6d0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/","bef"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","4f8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","3a8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","385"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","f4f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","bf9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/","89c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","fc9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","80d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/","b09"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","12f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/","86b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","25e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","c31"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/","1a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/","270"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/","c9a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","fdc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9cf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","a7b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","641"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","6e0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","8d1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/","50f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","d9a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/","ef5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/","43a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/","ed1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/","a4b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","a86"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","101"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/","5d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/","1e7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","88e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","d5d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","41e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/","d73"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/","267"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","95b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/","798"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/","0ec"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/","039"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","2fd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","8f3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","484"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","3d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/","caf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/","219"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","b35"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","c0f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/","fae"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","3c9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","71e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","e9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/next/Get Started/Create a Spark Cluster/","37f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Install SynapseML/",component:u("/SynapseML/docs/next/Get Started/Install SynapseML/","975"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/next/Get Started/Quickstart - Your First Models/","8a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/next/Get Started/Set up Cognitive Services/","47d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Overview/",component:u("/SynapseML/docs/next/Overview/","9a5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/","c47"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/","6e1"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_core/","7e6"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/","5eb"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/","49f"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/","443"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_core/","0a9"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/","137"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/","3f4"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/","d93"),exact:!0},{path:"/SynapseML/docs/next/Reference/Contributor Guide/",component:u("/SynapseML/docs/next/Reference/Contributor Guide/","dec"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Developer Setup/",component:u("/SynapseML/docs/next/Reference/Developer Setup/","1c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Docker Setup/",component:u("/SynapseML/docs/next/Reference/Docker Setup/","598"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Dotnet Setup/",component:u("/SynapseML/docs/next/Reference/Dotnet Setup/","ded"),exact:!0},{path:"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/","8a8"),exact:!0},{path:"/SynapseML/docs/next/Reference/R Setup/",component:u("/SynapseML/docs/next/Reference/R Setup/","d8b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/next/Use with MLFlow/Autologging/","0ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Install/",component:u("/SynapseML/docs/next/Use with MLFlow/Install/","9b7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Overview/",component:u("/SynapseML/docs/next/Use with MLFlow/Overview/","b10"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/",component:u("/SynapseML/docs/","352"),routes:[{path:"/SynapseML/docs/Deploy Models/Overview/",component:u("/SynapseML/docs/Deploy Models/Overview/","438"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/","e9f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","3f1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/","c3c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","de7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Overview/","52c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","dc7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/","2c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","f90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","425"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","d37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5e1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","eb1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","7e3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/","75a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","b1b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","cf3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/","bcf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","3e2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/","618"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","7c2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","a7a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/","1ba"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/","628"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/","90b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","57b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","b77"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","a81"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","93c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","87a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","699"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/Explore Algorithms/LightGBM/Overview/","300"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","5c1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/","06d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/","3b0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/","b8b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","d6d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","baa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/","179"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/","4b2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","54a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","572"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","743"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/","82e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/","dc2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","72f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/","e8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/","e6c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/","f44"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","9d9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","082"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","f62"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","804"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/","60b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/","3c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","32b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","046"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/","5b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","490"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","5dd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","804"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/Get Started/Create a Spark Cluster/","37d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Install SynapseML/",component:u("/SynapseML/docs/Get Started/Install SynapseML/","16c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/Get Started/Quickstart - Your First Models/","e0c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/Get Started/Set up Cognitive Services/","ea8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Overview/",component:u("/SynapseML/docs/Overview/","58e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_causal/","e9d"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/","419"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_core/","63c"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/","d43"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_vw/","bd9"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/","bbd"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_core/","1a1"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/","483"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_opencv/","8a0"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_vw/","fd7"),exact:!0},{path:"/SynapseML/docs/Reference/Contributor Guide/",component:u("/SynapseML/docs/Reference/Contributor Guide/","6e9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Developer Setup/",component:u("/SynapseML/docs/Reference/Developer Setup/","367"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Docker Setup/",component:u("/SynapseML/docs/Reference/Docker Setup/","a2a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Dotnet Setup/",component:u("/SynapseML/docs/Reference/Dotnet Setup/","995"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/","e58"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/R Setup/",component:u("/SynapseML/docs/Reference/R Setup/","d4d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/Use with MLFlow/Autologging/","1cd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Install/",component:u("/SynapseML/docs/Use with MLFlow/Install/","a7e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Overview/",component:u("/SynapseML/docs/Use with MLFlow/Overview/","424"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/",component:u("/SynapseML/","6e1"),exact:!0},{path:"*",component:u("*")}]},98934:(e,t,n)=>{"use strict";n.d(t,{_:()=>r,t:()=>o});var s=n(67294);const r=s.createContext(!1);function o(e){let{children:t}=e;const[n,o]=(0,s.useState)(!1);return(0,s.useEffect)((()=>{o(!0)}),[]),s.createElement(r.Provider,{value:n},t)}},49383:(e,t,n)=>{"use strict";var s=n(67294),r=n(73935),o=n(73727),a=n(70405),i=n(10412);const c=[n(56657),n(32497),n(3310),n(18320),n(52295)];var l=n(723),d=n(16550),p=n(18790);function u(e){let{children:t}=e;return s.createElement(s.Fragment,null,t)}var m=n(83117),h=n(35742),g=n(52263),f=n(44996),b=n(86668),y=n(10833),x=n(94711),v=n(19727),A=n(43320),S=n(90197);function E(){const{i18n:{defaultLocale:e,localeConfigs:t}}=(0,g.Z)(),n=(0,x.l)();return s.createElement(h.Z,null,Object.entries(t).map((e=>{let[t,{htmlLang:r}]=e;return s.createElement("link",{key:t,rel:"alternate",href:n.createUrl({locale:t,fullyQualified:!0}),hrefLang:r})})),s.createElement("link",{rel:"alternate",href:n.createUrl({locale:e,fullyQualified:!0}),hrefLang:"x-default"}))}function L(e){let{permalink:t}=e;const{siteConfig:{url:n}}=(0,g.Z)(),r=function(){const{siteConfig:{url:e}}=(0,g.Z)(),{pathname:t}=(0,d.TH)();return e+(0,f.Z)(t)}(),o=t?""+n+t:r;return s.createElement(h.Z,null,s.createElement("meta",{property:"og:url",content:o}),s.createElement("link",{rel:"canonical",href:o}))}function M(){const{i18n:{currentLocale:e}}=(0,g.Z)(),{metadata:t,image:n}=(0,b.L)();return s.createElement(s.Fragment,null,s.createElement(h.Z,null,s.createElement("meta",{name:"twitter:card",content:"summary_large_image"}),s.createElement("body",{className:v.h})),n&&s.createElement(y.d,{image:n}),s.createElement(L,null),s.createElement(E,null),s.createElement(S.Z,{tag:A.HX,locale:e}),s.createElement(h.Z,null,t.map(((e,t)=>s.createElement("meta",(0,m.Z)({key:t},e))))))}const k=new Map;function _(e){if(k.has(e.pathname))return{...e,pathname:k.get(e.pathname)};if((0,p.f)(l.Z,e.pathname).some((e=>{let{route:t}=e;return!0===t.exact})))return k.set(e.pathname,e.pathname),e;const t=e.pathname.trim().replace(/(?:\/index)?\.html$/,"")||"/";return k.set(e.pathname,t),{...e,pathname:t}}var w=n(98934),I=n(58940);function C(e){for(var t=arguments.length,n=new Array(t>1?t-1:0),s=1;s<t;s++)n[s-1]=arguments[s];const r=c.map((t=>{var s,r;const o=null!=(s=null==(r=t.default)?void 0:r[e])?s:t[e];return null==o?void 0:o(...n)}));return()=>r.forEach((e=>null==e?void 0:e()))}const Q=function(e){let{children:t,location:n,previousLocation:r}=e;return(0,s.useLayoutEffect)((()=>{r!==n&&(!function(e){let{location:t,previousLocation:n}=e;if(!n)return;const s=t.pathname===n.pathname,r=t.hash===n.hash,o=t.search===n.search;if(s&&r&&!o)return;const{hash:a}=t;if(a){const e=decodeURIComponent(a.substring(1)),t=document.getElementById(e);null==t||t.scrollIntoView()}else window.scrollTo(0,0)}({location:n,previousLocation:r}),C("onRouteDidUpdate",{previousLocation:r,location:n}))}),[r,n]),t};function R(e){const t=Array.from(new Set([e,decodeURI(e)])).map((e=>(0,p.f)(l.Z,e))).flat();return Promise.all(t.map((e=>null==e.route.component.preload?void 0:e.route.component.preload())))}class D extends s.Component{constructor(e){super(e),this.previousLocation=void 0,this.routeUpdateCleanupCb=void 0,this.previousLocation=null,this.routeUpdateCleanupCb=i.Z.canUseDOM?C("onRouteUpdate",{previousLocation:null,location:this.props.location}):()=>{},this.state={nextRouteHasLoaded:!0}}shouldComponentUpdate(e,t){if(e.location===this.props.location)return t.nextRouteHasLoaded;const n=e.location;return this.previousLocation=this.props.location,this.setState({nextRouteHasLoaded:!1}),this.routeUpdateCleanupCb=C("onRouteUpdate",{previousLocation:this.previousLocation,location:n}),R(n.pathname).then((()=>{this.routeUpdateCleanupCb(),this.setState({nextRouteHasLoaded:!0})})).catch((e=>{console.warn(e),window.location.reload()})),!1}render(){const{children:e,location:t}=this.props;return s.createElement(Q,{previousLocation:this.previousLocation,location:t},s.createElement(d.AW,{location:t,render:()=>e}))}}const T=D,O="__docusaurus-base-url-issue-banner-container",P="__docusaurus-base-url-issue-banner-suggestion-container",F="__DOCUSAURUS_INSERT_BASEURL_BANNER";function N(e){return"\nwindow['"+F+"'] = true;\n\ndocument.addEventListener('DOMContentLoaded', maybeInsertBanner);\n\nfunction maybeInsertBanner() {\n  var shouldInsert = window['"+F+"'];\n  shouldInsert && insertBanner();\n}\n\nfunction insertBanner() {\n  var bannerContainer = document.getElementById('"+O+"');\n  if (!bannerContainer) {\n    return;\n  }\n  var bannerHtml = "+JSON.stringify(function(e){return'\n<div id="__docusaurus-base-url-issue-banner" style="border: thick solid red; background-color: rgb(255, 230, 179); margin: 20px; padding: 20px; font-size: 20px;">\n   <p style="font-weight: bold; font-size: 30px;">Your Docusaurus site did not load properly.</p>\n   <p>A very common reason is a wrong site <a href="https://docusaurus.io/docs/docusaurus.config.js/#baseUrl" style="font-weight: bold;">baseUrl configuration</a>.</p>\n   <p>Current configured baseUrl = <span style="font-weight: bold; color: red;">'+e+"</span> "+("/"===e?" (default value)":"")+'</p>\n   <p>We suggest trying baseUrl = <span id="'+P+'" style="font-weight: bold; color: green;"></span></p>\n</div>\n'}(e)).replace(/</g,"\\<")+";\n  bannerContainer.innerHTML = bannerHtml;\n  var suggestionContainer = document.getElementById('"+P+"');\n  var actualHomePagePath = window.location.pathname;\n  var suggestedBaseUrl = actualHomePagePath.substr(-1) === '/'\n        ? actualHomePagePath\n        : actualHomePagePath + '/';\n  suggestionContainer.innerHTML = suggestedBaseUrl;\n}\n"}function B(){const{siteConfig:{baseUrl:e}}=(0,g.Z)();return(0,s.useLayoutEffect)((()=>{window[F]=!1}),[]),s.createElement(s.Fragment,null,!i.Z.canUseDOM&&s.createElement(h.Z,null,s.createElement("script",null,N(e))),s.createElement("div",{id:O}))}function V(){const{siteConfig:{baseUrl:e,baseUrlIssueBanner:t}}=(0,g.Z)(),{pathname:n}=(0,d.TH)();return t&&n===e?s.createElement(B,null):null}function G(){const{siteConfig:{favicon:e,title:t,noIndex:n},i18n:{currentLocale:r,localeConfigs:o}}=(0,g.Z)(),a=(0,f.Z)(e),{htmlLang:i,direction:c}=o[r];return s.createElement(h.Z,null,s.createElement("html",{lang:i,dir:c}),s.createElement("title",null,t),s.createElement("meta",{property:"og:title",content:t}),s.createElement("meta",{name:"viewport",content:"width=device-width, initial-scale=1.0"}),n&&s.createElement("meta",{name:"robots",content:"noindex, nofollow"}),e&&s.createElement("link",{rel:"icon",href:a}))}var W=n(44763);function U(){const e=(0,p.H)(l.Z),t=(0,d.TH)();return s.createElement(W.Z,null,s.createElement(I.M,null,s.createElement(w.t,null,s.createElement(u,null,s.createElement(G,null),s.createElement(M,null),s.createElement(V,null),s.createElement(T,{location:_(t)},e)))))}var z=n(16887);const H=function(e){try{return document.createElement("link").relList.supports(e)}catch{return!1}}("prefetch")?function(e){return new Promise(((t,n)=>{var s,r;if("undefined"==typeof document)return void n();const o=document.createElement("link");o.setAttribute("rel","prefetch"),o.setAttribute("href",e),o.onload=()=>t(),o.onerror=()=>n();const a=null!=(s=document.getElementsByTagName("head")[0])?s:null==(r=document.getElementsByName("script")[0])?void 0:r.parentNode;null==a||a.appendChild(o)}))}:function(e){return new Promise(((t,n)=>{const s=new XMLHttpRequest;s.open("GET",e,!0),s.withCredentials=!0,s.onload=()=>{200===s.status?t():n()},s.send(null)}))};var j=n(99670);const $=new Set,Z=new Set,K=()=>{var e,t;return(null==(e=navigator.connection)?void 0:e.effectiveType.includes("2g"))||(null==(t=navigator.connection)?void 0:t.saveData)},X={prefetch(e){if(!(e=>!K()&&!Z.has(e)&&!$.has(e))(e))return!1;$.add(e);const t=(0,p.f)(l.Z,e).flatMap((e=>{return t=e.route.path,Object.entries(z).filter((e=>{let[n]=e;return n.replace(/-[^-]+$/,"")===t})).flatMap((e=>{let[,t]=e;return Object.values((0,j.Z)(t))}));var t}));return Promise.all(t.map((e=>{const t=n.gca(e);return t&&!t.includes("undefined")?H(t).catch((()=>{})):Promise.resolve()})))},preload:e=>!!(e=>!K()&&!Z.has(e))(e)&&(Z.add(e),R(e))},Y=Object.freeze(X);if(i.Z.canUseDOM){window.docusaurus=Y;const e=r.hydrate;R(window.location.pathname).then((()=>{e(s.createElement(a.B6,null,s.createElement(o.VK,null,s.createElement(U,null))),document.getElementById("__docusaurus"))}))}},58940:(e,t,n)=>{"use strict";n.d(t,{_:()=>d,M:()=>p});var s=n(67294),r=n(36809);const o=JSON.parse('{"docusaurus-plugin-google-gtag":{"default":{"trackingID":["G-RWPE0183E8"],"anonymizeIP":true,"id":"default"}},"docusaurus-plugin-content-docs":{"default":{"path":"/SynapseML/docs","versions":[{"name":"current","label":"Next","isLast":false,"path":"/SynapseML/docs/next","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/next/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/next/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/next/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/next/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/next/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/next/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/next/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/next/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/next/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/next/Reference/Dotnet Setup"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet"},{"id":"Reference/R Setup","path":"/SynapseML/docs/next/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/next/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/next/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/next/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/next/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.4","label":"1.0.4","isLast":true,"path":"/SynapseML/docs","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.3","label":"1.0.3","isLast":false,"path":"/SynapseML/docs/1.0.3","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.3/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.3/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.3/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.3/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.3/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.3/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.3/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.3/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.3/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.2","label":"1.0.2","isLast":false,"path":"/SynapseML/docs/1.0.2","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.2/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.2/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.2/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.2/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.2/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.2/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.2/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.2/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.2/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.1","label":"1.0.1","isLast":false,"path":"/SynapseML/docs/1.0.1","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.1/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.1/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.1/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.1/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.1/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.1/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.1/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.1/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.1/Overview","label":"What is SynapseML?"}}}},{"name":"0.11.4","label":"0.11.4","isLast":false,"path":"/SynapseML/docs/0.11.4","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/0.11.4/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/0.11.4/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/0.11.4/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/0.11.4/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/0.11.4/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/0.11.4/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/0.11.4/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/0.11.4/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/0.11.4/Overview","label":"What is SynapseML?"}}}},{"name":"0.11.3","label":"0.11.3","isLast":false,"path":"/SynapseML/docs/0.11.3","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/0.11.3/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/0.11.3/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/0.11.3/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/0.11.3/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/0.11.3/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/0.11.3/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/0.11.3/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/0.11.3/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/0.11.3/Overview","label":"What is SynapseML?"}}}}],"breadcrumbs":true}}}'),a=JSON.parse('{"defaultLocale":"en","locales":["en"],"path":"i18n","currentLocale":"en","localeConfigs":{"en":{"label":"English","direction":"ltr","htmlLang":"en","calendar":"gregory","path":"en"}}}');var i=n(57529);const c=JSON.parse('{"docusaurusVersion":"2.4.1","siteVersion":"0.0.0","pluginVersions":{"docusaurus-plugin-content-docs":{"type":"package","name":"@docusaurus/plugin-content-docs","version":"2.4.1"},"docusaurus-plugin-content-blog":{"type":"package","name":"@docusaurus/plugin-content-blog","version":"2.4.1"},"docusaurus-plugin-content-pages":{"type":"package","name":"@docusaurus/plugin-content-pages","version":"2.4.1"},"docusaurus-plugin-google-gtag":{"type":"package","name":"@docusaurus/plugin-google-gtag","version":"2.4.1"},"docusaurus-plugin-sitemap":{"type":"package","name":"@docusaurus/plugin-sitemap","version":"2.4.1"},"docusaurus-theme-classic":{"type":"package","name":"@docusaurus/theme-classic","version":"2.4.1"},"docusaurus-theme-search-algolia":{"type":"package","name":"@docusaurus/theme-search-algolia","version":"2.4.1"},"docusaurus-plugin-client-redirects":{"type":"package","name":"@docusaurus/plugin-client-redirects","version":"2.4.1"}}}'),l={siteConfig:r.default,siteMetadata:c,globalData:o,i18n:a,codeTranslations:i},d=s.createContext(l);function p(e){let{children:t}=e;return s.createElement(d.Provider,{value:l},t)}},44763:(e,t,n)=>{"use strict";n.d(t,{Z:()=>u});var s=n(67294),r=n(10412),o=n(35742),a=n(18780),i=n(7452);function c(e){let{error:t,tryAgain:n}=e;return s.createElement("div",{style:{display:"flex",flexDirection:"column",justifyContent:"center",alignItems:"flex-start",minHeight:"100vh",width:"100%",maxWidth:"80ch",fontSize:"20px",margin:"0 auto",padding:"1rem"}},s.createElement("h1",{style:{fontSize:"3rem"}},"This page crashed"),s.createElement("button",{type:"button",onClick:n,style:{margin:"1rem 0",fontSize:"2rem",cursor:"pointer",borderRadius:20,padding:"1rem"}},"Try again"),s.createElement(l,{error:t}))}function l(e){let{error:t}=e;const n=(0,a.getErrorCausalChain)(t).map((e=>e.message)).join("\n\nCause:\n");return s.createElement("p",{style:{whiteSpace:"pre-wrap"}},n)}function d(e){let{error:t,tryAgain:n}=e;return s.createElement(u,{fallback:()=>s.createElement(c,{error:t,tryAgain:n})},s.createElement(o.Z,null,s.createElement("title",null,"Page Error")),s.createElement(i.Z,null,s.createElement(c,{error:t,tryAgain:n})))}const p=e=>s.createElement(d,e);class u extends s.Component{constructor(e){super(e),this.state={error:null}}componentDidCatch(e){r.Z.canUseDOM&&this.setState({error:e})}render(){const{children:e}=this.props,{error:t}=this.state;if(t){var n;const e={error:t,tryAgain:()=>this.setState({error:null})};return(null!=(n=this.props.fallback)?n:p)(e)}return null!=e?e:null}}},10412:(e,t,n)=>{"use strict";n.d(t,{Z:()=>r});const s="undefined"!=typeof window&&"document"in window&&"createElement"in window.document,r={canUseDOM:s,canUseEventListeners:s&&("addEventListener"in window||"attachEvent"in window),canUseIntersectionObserver:s&&"IntersectionObserver"in window,canUseViewport:s&&"screen"in window}},35742:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(70405);function o(e){return s.createElement(r.ql,e)}},39960:(e,t,n)=>{"use strict";n.d(t,{Z:()=>m});var s=n(83117),r=n(67294),o=n(73727),a=n(18780),i=n(52263),c=n(13919),l=n(10412);const d=r.createContext({collectLink:()=>{}});var p=n(44996);function u(e,t){var n,u;let{isNavLink:m,to:h,href:g,activeClassName:f,isActive:b,"data-noBrokenLinkCheck":y,autoAddBaseUrl:x=!0,...v}=e;const{siteConfig:{trailingSlash:A,baseUrl:S}}=(0,i.Z)(),{withBaseUrl:E}=(0,p.C)(),L=(0,r.useContext)(d),M=(0,r.useRef)(null);(0,r.useImperativeHandle)(t,(()=>M.current));const k=h||g;const _=(0,c.Z)(k),w=null==k?void 0:k.replace("pathname://","");let I=void 0!==w?(C=w,x&&(e=>e.startsWith("/"))(C)?E(C):C):void 0;var C;I&&_&&(I=(0,a.applyTrailingSlash)(I,{trailingSlash:A,baseUrl:S}));const Q=(0,r.useRef)(!1),R=m?o.OL:o.rU,D=l.Z.canUseIntersectionObserver,T=(0,r.useRef)(),O=()=>{Q.current||null==I||(window.docusaurus.preload(I),Q.current=!0)};(0,r.useEffect)((()=>(!D&&_&&null!=I&&window.docusaurus.prefetch(I),()=>{D&&T.current&&T.current.disconnect()})),[T,I,D,_]);const P=null!=(n=null==(u=I)?void 0:u.startsWith("#"))&&n,F=!I||!_||P;return F||y||L.collectLink(I),F?r.createElement("a",(0,s.Z)({ref:M,href:I},k&&!_&&{target:"_blank",rel:"noopener noreferrer"},v)):r.createElement(R,(0,s.Z)({},v,{onMouseEnter:O,onTouchStart:O,innerRef:e=>{M.current=e,D&&e&&_&&(T.current=new window.IntersectionObserver((t=>{t.forEach((t=>{e===t.target&&(t.isIntersecting||t.intersectionRatio>0)&&(T.current.unobserve(e),T.current.disconnect(),null!=I&&window.docusaurus.prefetch(I))}))})),T.current.observe(e))},to:I},m&&{isActive:b,activeClassName:f}))}const m=r.forwardRef(u)},95999:(e,t,n)=>{"use strict";n.d(t,{Z:()=>c,I:()=>i});var s=n(67294);function r(e,t){const n=e.split(/(\{\w+\})/).map(((e,n)=>{if(n%2==1){const n=null==t?void 0:t[e.slice(1,-1)];if(void 0!==n)return n}return e}));return n.some((e=>(0,s.isValidElement)(e)))?n.map(((e,t)=>(0,s.isValidElement)(e)?s.cloneElement(e,{key:t}):e)).filter((e=>""!==e)):n.join("")}var o=n(57529);function a(e){var t,n;let{id:s,message:r}=e;if(void 0===s&&void 0===r)throw new Error("Docusaurus translation declarations must have at least a translation id or a default translation message");return null!=(t=null!=(n=o[null!=s?s:r])?n:r)?t:s}function i(e,t){let{message:n,id:s}=e;return r(a({message:n,id:s}),t)}function c(e){let{children:t,id:n,values:o}=e;if(t&&"string"!=typeof t)throw console.warn("Illegal <Translate> children",t),new Error("The Docusaurus <Translate> component only accept simple string values");const i=a({message:t,id:n});return s.createElement(s.Fragment,null,r(i,o))}},29935:(e,t,n)=>{"use strict";n.d(t,{m:()=>s});const s="default"},13919:(e,t,n)=>{"use strict";function s(e){return/^(?:\w*:|\/\/)/.test(e)}function r(e){return void 0!==e&&!s(e)}n.d(t,{Z:()=>r,b:()=>s})},44996:(e,t,n)=>{"use strict";n.d(t,{C:()=>a,Z:()=>i});var s=n(67294),r=n(52263),o=n(13919);function a(){const{siteConfig:{baseUrl:e,url:t}}=(0,r.Z)(),n=(0,s.useCallback)(((n,s)=>function(e,t,n,s){let{forcePrependBaseUrl:r=!1,absolute:a=!1}=void 0===s?{}:s;if(!n||n.startsWith("#")||(0,o.b)(n))return n;if(r)return t+n.replace(/^\//,"");if(n===t.replace(/\/$/,""))return t;const i=n.startsWith(t)?n:t+n.replace(/^\//,"");return a?e+i:i}(t,e,n,s)),[t,e]);return{withBaseUrl:n}}function i(e,t){void 0===t&&(t={});const{withBaseUrl:n}=a();return n(e,t)}},52263:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(58940);function o(){return(0,s.useContext)(r._)}},72389:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(98934);function o(){return(0,s.useContext)(r._)}},99670:(e,t,n)=>{"use strict";n.d(t,{Z:()=>s});function s(e){const t={};return function e(n,s){Object.entries(n).forEach((n=>{let[r,o]=n;const a=s?s+"."+r:r;var i;"object"==typeof(i=o)&&i&&Object.keys(i).length>0?e(o,a):t[a]=o}))}(e),t}},30226:(e,t,n)=>{"use strict";n.d(t,{_:()=>r,z:()=>o});var s=n(67294);const r=s.createContext(null);function o(e){let{children:t,value:n}=e;const o=s.useContext(r),a=(0,s.useMemo)((()=>function(e){let{parent:t,value:n}=e;if(!t){if(!n)throw new Error("Unexpected: no Docusaurus route context found");if(!("plugin"in n))throw new Error("Unexpected: Docusaurus topmost route context has no `plugin` attribute");return n}const s={...t.data,...null==n?void 0:n.data};return{plugin:t.plugin,data:s}}({parent:o,value:n})),[o,n]);return s.createElement(r.Provider,{value:a},t)}},80143:(e,t,n)=>{"use strict";n.d(t,{Iw:()=>b,gA:()=>m,WS:()=>h,_r:()=>p,Jo:()=>y,zh:()=>u,yW:()=>f,gB:()=>g});var s=n(16550),r=n(52263),o=n(29935);function a(e,t){void 0===t&&(t={});const n=function(){const{globalData:e}=(0,r.Z)();return e}()[e];if(!n&&t.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin.');return n}const i=e=>e.versions.find((e=>e.isLast));function c(e,t){const n=i(e);return[...e.versions.filter((e=>e!==n)),n].find((e=>!!(0,s.LX)(t,{path:e.path,exact:!1,strict:!1})))}function l(e,t){const n=c(e,t),r=null==n?void 0:n.docs.find((e=>!!(0,s.LX)(t,{path:e.path,exact:!0,strict:!1})));return{activeVersion:n,activeDoc:r,alternateDocVersions:r?function(t){const n={};return e.versions.forEach((e=>{e.docs.forEach((s=>{s.id===t&&(n[e.name]=s)}))})),n}(r.id):{}}}const d={},p=()=>{var e;return null!=(e=a("docusaurus-plugin-content-docs"))?e:d},u=e=>function(e,t,n){void 0===t&&(t=o.m),void 0===n&&(n={});const s=a(e),r=null==s?void 0:s[t];if(!r&&n.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin with id "'+t+'".');return r}("docusaurus-plugin-content-docs",e,{failfast:!0});function m(e){void 0===e&&(e={});const t=p(),{pathname:n}=(0,s.TH)();return function(e,t,n){void 0===n&&(n={});const r=Object.entries(e).sort(((e,t)=>t[1].path.localeCompare(e[1].path))).find((e=>{let[,n]=e;return!!(0,s.LX)(t,{path:n.path,exact:!1,strict:!1})})),o=r?{pluginId:r[0],pluginData:r[1]}:void 0;if(!o&&n.failfast)throw new Error("Can't find active docs plugin for \""+t+'" pathname, while it was expected to be found. Maybe you tried to use a docs feature that can only be used on a docs-related page? Existing docs plugin paths are: '+Object.values(e).map((e=>e.path)).join(", "));return o}(t,n,e)}function h(e){void 0===e&&(e={});const t=m(e),{pathname:n}=(0,s.TH)();if(!t)return;return{activePlugin:t,activeVersion:c(t.pluginData,n)}}function g(e){return u(e).versions}function f(e){const t=u(e);return i(t)}function b(e){const t=u(e),{pathname:n}=(0,s.TH)();return l(t,n)}function y(e){const t=u(e),{pathname:n}=(0,s.TH)();return function(e,t){const n=i(e);return{latestDocSuggestion:l(e,t).alternateDocVersions[n.name],latestVersionSuggestion:n}}(t,n)}},56657:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>s});const s={onRouteDidUpdate(e){let{location:t,previousLocation:n}=e;!n||t.pathname===n.pathname&&t.search===n.search&&t.hash===n.hash||setTimeout((()=>{window.gtag("event","page_view",{page_title:document.title,page_location:window.location.href,page_path:t.pathname+t.search+t.hash})}))}}},18320:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>o});var s=n(74865),r=n.n(s);r().configure({showSpinner:!1});const o={onRouteUpdate(e){let{location:t,previousLocation:n}=e;if(n&&t.pathname!==n.pathname){const e=window.setTimeout((()=>{r().start()}),200);return()=>window.clearTimeout(e)}},onRouteDidUpdate(){r().done()}}},3310:(e,t,n)=>{"use strict";n.r(t);var s=n(87410),r=n(36809);!function(e){const{themeConfig:{prism:t}}=r.default,{additionalLanguages:s}=t;globalThis.Prism=e,s.forEach((e=>{n(55979)("./prism-"+e)})),delete globalThis.Prism}(s.Z)},39471:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294);const r="iconExternalLink_nPIU";function o(e){let{width:t=13.5,height:n=13.5}=e;return s.createElement("svg",{width:t,height:n,"aria-hidden":"true",viewBox:"0 0 24 24",className:r},s.createElement("path",{fill:"currentColor",d:"M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"}))}},7452:(e,t,n)=>{"use strict";n.d(t,{Z:()=>Dt});var s=n(67294),r=n(86010),o=n(44763),a=n(10833),i=n(83117),c=n(16550),l=n(95999),d=n(85936);const p="__docusaurus_skipToContent_fallback";function u(e){e.setAttribute("tabindex","-1"),e.focus(),e.removeAttribute("tabindex")}function m(){const e=(0,s.useRef)(null),{action:t}=(0,c.k6)(),n=(0,s.useCallback)((e=>{e.preventDefault();const t=null!=(n=document.querySelector("main:first-of-type"))?n:document.getElementById(p);var n;t&&u(t)}),[]);return(0,d.S)((n=>{let{location:s}=n;e.current&&!s.hash&&"PUSH"===t&&u(e.current)})),{containerRef:e,onClick:n}}const h=(0,l.I)({id:"theme.common.skipToMainContent",description:"The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation",message:"Skip to main content"});function g(e){var t;const n=null!=(t=e.children)?t:h,{containerRef:r,onClick:o}=m();return s.createElement("div",{ref:r,role:"region","aria-label":h},s.createElement("a",(0,i.Z)({},e,{href:"#"+p,onClick:o}),n))}var f=n(35281),b=n(19727);const y="skipToContent_fXgn";function x(){return s.createElement(g,{className:y})}var v=n(86668),A=n(59689);function S(e){let{width:t=21,height:n=21,color:r="currentColor",strokeWidth:o=1.2,className:a,...c}=e;return s.createElement("svg",(0,i.Z)({viewBox:"0 0 15 15",width:t,height:n},c),s.createElement("g",{stroke:r,strokeWidth:o},s.createElement("path",{d:"M.75.75l13.5 13.5M14.25.75L.75 14.25"})))}const E="closeButton_CVFx";function L(e){return s.createElement("button",(0,i.Z)({type:"button","aria-label":(0,l.I)({id:"theme.AnnouncementBar.closeButtonAriaLabel",message:"Close",description:"The ARIA label for close button of announcement bar"})},e,{className:(0,r.Z)("clean-btn close",E,e.className)}),s.createElement(S,{width:14,height:14,strokeWidth:3.1}))}const M="content_knG7";function k(e){const{announcementBar:t}=(0,v.L)(),{content:n}=t;return s.createElement("div",(0,i.Z)({},e,{className:(0,r.Z)(M,e.className),dangerouslySetInnerHTML:{__html:n}}))}const _="announcementBar_mb4j",w="announcementBarPlaceholder_vyr4",I="announcementBarClose_gvF7",C="announcementBarContent_xLdY";function Q(){const{announcementBar:e}=(0,v.L)(),{isActive:t,close:n}=(0,A.nT)();if(!t)return null;const{backgroundColor:r,textColor:o,isCloseable:a}=e;return s.createElement("div",{className:_,style:{backgroundColor:r,color:o},role:"banner"},a&&s.createElement("div",{className:w}),s.createElement(k,{className:C}),a&&s.createElement(L,{onClick:n,className:I}))}var R=n(93163),D=n(12466);var T=n(902),O=n(13102);const P=s.createContext(null);function F(e){let{children:t}=e;const n=function(){const e=(0,R.e)(),t=(0,O.HY)(),[n,r]=(0,s.useState)(!1),o=null!==t.component,a=(0,T.D9)(o);return(0,s.useEffect)((()=>{o&&!a&&r(!0)}),[o,a]),(0,s.useEffect)((()=>{o?e.shown||r(!0):r(!1)}),[e.shown,o]),(0,s.useMemo)((()=>[n,r]),[n])}();return s.createElement(P.Provider,{value:n},t)}function N(e){if(e.component){const t=e.component;return s.createElement(t,e.props)}}function B(){const e=(0,s.useContext)(P);if(!e)throw new T.i6("NavbarSecondaryMenuDisplayProvider");const[t,n]=e,r=(0,s.useCallback)((()=>n(!1)),[n]),o=(0,O.HY)();return(0,s.useMemo)((()=>({shown:t,hide:r,content:N(o)})),[r,o,t])}function V(e){let{header:t,primaryMenu:n,secondaryMenu:o}=e;const{shown:a}=B();return s.createElement("div",{className:"navbar-sidebar"},t,s.createElement("div",{className:(0,r.Z)("navbar-sidebar__items",{"navbar-sidebar__items--show-secondary":a})},s.createElement("div",{className:"navbar-sidebar__item menu"},n),s.createElement("div",{className:"navbar-sidebar__item menu"},o)))}var G=n(92949),W=n(72389);function U(e){return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:24,height:24},e),s.createElement("path",{fill:"currentColor",d:"M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"}))}function z(e){return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:24,height:24},e),s.createElement("path",{fill:"currentColor",d:"M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"}))}const H={toggle:"toggle_vylO",toggleButton:"toggleButton_gllP",darkToggleIcon:"darkToggleIcon_wfgR",lightToggleIcon:"lightToggleIcon_pyhR",toggleButtonDisabled:"toggleButtonDisabled_aARS"};function j(e){let{className:t,buttonClassName:n,value:o,onChange:a}=e;const i=(0,W.Z)(),c=(0,l.I)({message:"Switch between dark and light mode (currently {mode})",id:"theme.colorToggle.ariaLabel",description:"The ARIA label for the navbar color mode toggle"},{mode:"dark"===o?(0,l.I)({message:"dark mode",id:"theme.colorToggle.ariaLabel.mode.dark",description:"The name for the dark color mode"}):(0,l.I)({message:"light mode",id:"theme.colorToggle.ariaLabel.mode.light",description:"The name for the light color mode"})});return s.createElement("div",{className:(0,r.Z)(H.toggle,t)},s.createElement("button",{className:(0,r.Z)("clean-btn",H.toggleButton,!i&&H.toggleButtonDisabled,n),type:"button",onClick:()=>a("dark"===o?"light":"dark"),disabled:!i,title:c,"aria-label":c,"aria-live":"polite"},s.createElement(U,{className:(0,r.Z)(H.toggleIcon,H.lightToggleIcon)}),s.createElement(z,{className:(0,r.Z)(H.toggleIcon,H.darkToggleIcon)})))}const $=s.memo(j),Z="darkNavbarColorModeToggle_X3D1";function K(e){let{className:t}=e;const n=(0,v.L)().navbar.style,r=(0,v.L)().colorMode.disableSwitch,{colorMode:o,setColorMode:a}=(0,G.I)();return r?null:s.createElement($,{className:t,buttonClassName:"dark"===n?Z:void 0,value:o,onChange:a})}var X=n(21327);function Y(){return s.createElement(X.Z,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:"navbar__title text--truncate"})}function q(){const e=(0,R.e)();return s.createElement("button",{type:"button","aria-label":(0,l.I)({id:"theme.docs.sidebar.closeSidebarButtonAriaLabel",message:"Close navigation bar",description:"The ARIA label for close button of mobile sidebar"}),className:"clean-btn navbar-sidebar__close",onClick:()=>e.toggle()},s.createElement(S,{color:"var(--ifm-color-emphasis-600)"}))}function J(){return s.createElement("div",{className:"navbar-sidebar__brand"},s.createElement(Y,null),s.createElement(K,{className:"margin-right--md"}),s.createElement(q,null))}var ee=n(39960),te=n(44996),ne=n(13919),se=n(98022),re=n(39471);function oe(e){let{activeBasePath:t,activeBaseRegex:n,to:r,href:o,label:a,html:c,isDropdownLink:l,prependBaseUrlToHref:d,...p}=e;const u=(0,te.Z)(r),m=(0,te.Z)(t),h=(0,te.Z)(o,{forcePrependBaseUrl:!0}),g=a&&o&&!(0,ne.Z)(o),f=c?{dangerouslySetInnerHTML:{__html:c}}:{children:s.createElement(s.Fragment,null,a,g&&s.createElement(re.Z,l&&{width:12,height:12}))};return o?s.createElement(ee.Z,(0,i.Z)({href:d?h:o},p,f)):s.createElement(ee.Z,(0,i.Z)({to:u,isNavLink:!0},(t||n)&&{isActive:(e,t)=>n?(0,se.F)(n,t.pathname):t.pathname.startsWith(m)},p,f))}function ae(e){let{className:t,isDropdownItem:n=!1,...o}=e;const a=s.createElement(oe,(0,i.Z)({className:(0,r.Z)(n?"dropdown__link":"navbar__item navbar__link",t),isDropdownLink:n},o));return n?s.createElement("li",null,a):a}function ie(e){let{className:t,isDropdownItem:n,...o}=e;return s.createElement("li",{className:"menu__list-item"},s.createElement(oe,(0,i.Z)({className:(0,r.Z)("menu__link",t)},o)))}function ce(e){var t;let{mobile:n=!1,position:r,...o}=e;const a=n?ie:ae;return s.createElement(a,(0,i.Z)({},o,{activeClassName:null!=(t=o.activeClassName)?t:n?"menu__link--active":"navbar__link--active"}))}var le=n(86043),de=n(48596),pe=n(52263);function ue(e,t){return e.some((e=>function(e,t){return!!(0,de.Mg)(e.to,t)||!!(0,se.F)(e.activeBaseRegex,t)||!(!e.activeBasePath||!t.startsWith(e.activeBasePath))}(e,t)))}function me(e){var t;let{items:n,position:o,className:a,onClick:c,...l}=e;const d=(0,s.useRef)(null),[p,u]=(0,s.useState)(!1);return(0,s.useEffect)((()=>{const e=e=>{d.current&&!d.current.contains(e.target)&&u(!1)};return document.addEventListener("mousedown",e),document.addEventListener("touchstart",e),document.addEventListener("focusin",e),()=>{document.removeEventListener("mousedown",e),document.removeEventListener("touchstart",e),document.removeEventListener("focusin",e)}}),[d]),s.createElement("div",{ref:d,className:(0,r.Z)("navbar__item","dropdown","dropdown--hoverable",{"dropdown--right":"right"===o,"dropdown--show":p})},s.createElement(oe,(0,i.Z)({"aria-haspopup":"true","aria-expanded":p,role:"button",href:l.to?void 0:"#",className:(0,r.Z)("navbar__link",a)},l,{onClick:l.to?void 0:e=>e.preventDefault(),onKeyDown:e=>{"Enter"===e.key&&(e.preventDefault(),u(!p))}}),null!=(t=l.children)?t:l.label),s.createElement("ul",{className:"dropdown__menu"},n.map(((e,t)=>s.createElement(je,(0,i.Z)({isDropdownItem:!0,activeClassName:"dropdown__link--active"},e,{key:t}))))))}function he(e){var t;let{items:n,className:o,position:a,onClick:l,...d}=e;const p=function(){const{siteConfig:{baseUrl:e}}=(0,pe.Z)(),{pathname:t}=(0,c.TH)();return t.replace(e,"/")}(),u=ue(n,p),{collapsed:m,toggleCollapsed:h,setCollapsed:g}=(0,le.u)({initialState:()=>!u});return(0,s.useEffect)((()=>{u&&g(!u)}),[p,u,g]),s.createElement("li",{className:(0,r.Z)("menu__list-item",{"menu__list-item--collapsed":m})},s.createElement(oe,(0,i.Z)({role:"button",className:(0,r.Z)("menu__link menu__link--sublist menu__link--sublist-caret",o)},d,{onClick:e=>{e.preventDefault(),h()}}),null!=(t=d.children)?t:d.label),s.createElement(le.z,{lazy:!0,as:"ul",className:"menu__list",collapsed:m},n.map(((e,t)=>s.createElement(je,(0,i.Z)({mobile:!0,isDropdownItem:!0,onClick:l,activeClassName:"menu__link--active"},e,{key:t}))))))}function ge(e){let{mobile:t=!1,...n}=e;const r=t?he:me;return s.createElement(r,n)}var fe=n(94711);function be(e){let{width:t=20,height:n=20,...r}=e;return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:t,height:n,"aria-hidden":!0},r),s.createElement("path",{fill:"currentColor",d:"M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"}))}const ye="iconLanguage_nlXk";function xe(){return s.createElement("svg",{width:"15",height:"15",className:"DocSearch-Control-Key-Icon"},s.createElement("path",{d:"M4.505 4.496h2M5.505 5.496v5M8.216 4.496l.055 5.993M10 7.5c.333.333.5.667.5 1v2M12.326 4.5v5.996M8.384 4.496c1.674 0 2.116 0 2.116 1.5s-.442 1.5-2.116 1.5M3.205 9.303c-.09.448-.277 1.21-1.241 1.203C1 10.5.5 9.513.5 8V7c0-1.57.5-2.5 1.464-2.494.964.006 1.134.598 1.24 1.342M12.553 10.5h1.953",strokeWidth:"1.2",stroke:"currentColor",fill:"none",strokeLinecap:"square"}))}var ve=n(20830),Ae=["translations"];function Se(){return Se=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},Se.apply(this,arguments)}function Ee(e,t){return function(e){if(Array.isArray(e))return e}(e)||function(e,t){var n=null==e?null:"undefined"!=typeof Symbol&&e[Symbol.iterator]||e["@@iterator"];if(null==n)return;var s,r,o=[],a=!0,i=!1;try{for(n=n.call(e);!(a=(s=n.next()).done)&&(o.push(s.value),!t||o.length!==t);a=!0);}catch(c){i=!0,r=c}finally{try{a||null==n.return||n.return()}finally{if(i)throw r}}return o}(e,t)||function(e,t){if(!e)return;if("string"==typeof e)return Le(e,t);var n=Object.prototype.toString.call(e).slice(8,-1);"Object"===n&&e.constructor&&(n=e.constructor.name);if("Map"===n||"Set"===n)return Array.from(e);if("Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n))return Le(e,t)}(e,t)||function(){throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}()}function Le(e,t){(null==t||t>e.length)&&(t=e.length);for(var n=0,s=new Array(t);n<t;n++)s[n]=e[n];return s}function Me(e,t){if(null==e)return{};var n,s,r=function(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var ke="Ctrl";var _e=s.forwardRef((function(e,t){var n=e.translations,r=void 0===n?{}:n,o=Me(e,Ae),a=r.buttonText,i=void 0===a?"Search":a,c=r.buttonAriaLabel,l=void 0===c?"Search":c,d=Ee((0,s.useState)(null),2),p=d[0],u=d[1];return(0,s.useEffect)((function(){"undefined"!=typeof navigator&&(/(Mac|iPhone|iPod|iPad)/i.test(navigator.platform)?u("\u2318"):u(ke))}),[]),s.createElement("button",Se({type:"button",className:"DocSearch DocSearch-Button","aria-label":l},o,{ref:t}),s.createElement("span",{className:"DocSearch-Button-Container"},s.createElement(ve.W,null),s.createElement("span",{className:"DocSearch-Button-Placeholder"},i)),s.createElement("span",{className:"DocSearch-Button-Keys"},null!==p&&s.createElement(s.Fragment,null,s.createElement("kbd",{className:"DocSearch-Button-Key"},p===ke?s.createElement(xe,null):p),s.createElement("kbd",{className:"DocSearch-Button-Key"},"K"))))})),we=n(35742),Ie=n(66177),Ce=n(239),Qe=n(43320);var Re=n(73935);const De={button:{buttonText:(0,l.I)({id:"theme.SearchBar.label",message:"Search",description:"The ARIA label and placeholder for search button"}),buttonAriaLabel:(0,l.I)({id:"theme.SearchBar.label",message:"Search",description:"The ARIA label and placeholder for search button"})},modal:{searchBox:{resetButtonTitle:(0,l.I)({id:"theme.SearchModal.searchBox.resetButtonTitle",message:"Clear the query",description:"The label and ARIA label for search box reset button"}),resetButtonAriaLabel:(0,l.I)({id:"theme.SearchModal.searchBox.resetButtonTitle",message:"Clear the query",description:"The label and ARIA label for search box reset button"}),cancelButtonText:(0,l.I)({id:"theme.SearchModal.searchBox.cancelButtonText",message:"Cancel",description:"The label and ARIA label for search box cancel button"}),cancelButtonAriaLabel:(0,l.I)({id:"theme.SearchModal.searchBox.cancelButtonText",message:"Cancel",description:"The label and ARIA label for search box cancel button"})},startScreen:{recentSearchesTitle:(0,l.I)({id:"theme.SearchModal.startScreen.recentSearchesTitle",message:"Recent",description:"The title for recent searches"}),noRecentSearchesText:(0,l.I)({id:"theme.SearchModal.startScreen.noRecentSearchesText",message:"No recent searches",description:"The text when no recent searches"}),saveRecentSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.saveRecentSearchButtonTitle",message:"Save this search",description:"The label for save recent search button"}),removeRecentSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.removeRecentSearchButtonTitle",message:"Remove this search from history",description:"The label for remove recent search button"}),favoriteSearchesTitle:(0,l.I)({id:"theme.SearchModal.startScreen.favoriteSearchesTitle",message:"Favorite",description:"The title for favorite searches"}),removeFavoriteSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.removeFavoriteSearchButtonTitle",message:"Remove this search from favorites",description:"The label for remove favorite search button"})},errorScreen:{titleText:(0,l.I)({id:"theme.SearchModal.errorScreen.titleText",message:"Unable to fetch results",description:"The title for error screen of search modal"}),helpText:(0,l.I)({id:"theme.SearchModal.errorScreen.helpText",message:"You might want to check your network connection.",description:"The help text for error screen of search modal"})},footer:{selectText:(0,l.I)({id:"theme.SearchModal.footer.selectText",message:"to select",description:"The explanatory text of the action for the enter key"}),selectKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.selectKeyAriaLabel",message:"Enter key",description:"The ARIA label for the Enter key button that makes the selection"}),navigateText:(0,l.I)({id:"theme.SearchModal.footer.navigateText",message:"to navigate",description:"The explanatory text of the action for the Arrow up and Arrow down key"}),navigateUpKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.navigateUpKeyAriaLabel",message:"Arrow up",description:"The ARIA label for the Arrow up key button that makes the navigation"}),navigateDownKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.navigateDownKeyAriaLabel",message:"Arrow down",description:"The ARIA label for the Arrow down key button that makes the navigation"}),closeText:(0,l.I)({id:"theme.SearchModal.footer.closeText",message:"to close",description:"The explanatory text of the action for Escape key"}),closeKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.closeKeyAriaLabel",message:"Escape key",description:"The ARIA label for the Escape key button that close the modal"}),searchByText:(0,l.I)({id:"theme.SearchModal.footer.searchByText",message:"Search by",description:"The text explain that the search is making by Algolia"})},noResultsScreen:{noResultsText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.noResultsText",message:"No results for",description:"The text explains that there are no results for the following search"}),suggestedQueryText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.suggestedQueryText",message:"Try searching for",description:"The text for the suggested query when no results are found for the following search"}),reportMissingResultsText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.reportMissingResultsText",message:"Believe this query should return results?",description:"The text for the question where the user thinks there are missing results"}),reportMissingResultsLinkText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.reportMissingResultsLinkText",message:"Let us know.",description:"The text for the link to report missing results"})}},placeholder:(0,l.I)({id:"theme.SearchModal.placeholder",message:"Search docs",description:"The placeholder of the input of the DocSearch pop-up modal"})};let Te=null;function Oe(e){let{hit:t,children:n}=e;return s.createElement(ee.Z,{to:t.url},n)}function Pe(e){let{state:t,onClose:n}=e;const r=(0,Ie.M)();return s.createElement(ee.Z,{to:r(t.query),onClick:n},s.createElement(l.Z,{id:"theme.SearchBar.seeAll",values:{count:t.context.nbHits}},"See all {count} results"))}function Fe(e){var t,r;let{contextualSearch:o,externalUrlRegex:a,...l}=e;const{siteMetadata:d}=(0,pe.Z)(),p=(0,Ce.l)(),u=function(){const{locale:e,tags:t}=(0,Qe._q)();return["language:"+e,t.map((e=>"docusaurus_tag:"+e))]}(),m=null!=(t=null==(r=l.searchParameters)?void 0:r.facetFilters)?t:[],h=o?function(e,t){const n=e=>"string"==typeof e?[e]:e;return[...n(e),...n(t)]}(u,m):m,g={...l.searchParameters,facetFilters:h},f=(0,c.k6)(),b=(0,s.useRef)(null),y=(0,s.useRef)(null),[x,v]=(0,s.useState)(!1),[A,S]=(0,s.useState)(void 0),E=(0,s.useCallback)((()=>Te?Promise.resolve():Promise.all([n.e(76780).then(n.bind(n,76780)),Promise.all([n.e(40532),n.e(46945)]).then(n.bind(n,46945)),Promise.all([n.e(40532),n.e(18894)]).then(n.bind(n,18894))]).then((e=>{let[{DocSearchModal:t}]=e;Te=t}))),[]),L=(0,s.useCallback)((()=>{E().then((()=>{b.current=document.createElement("div"),document.body.insertBefore(b.current,document.body.firstChild),v(!0)}))}),[E,v]),M=(0,s.useCallback)((()=>{var e;v(!1),null==(e=b.current)||e.remove()}),[v]),k=(0,s.useCallback)((e=>{E().then((()=>{v(!0),S(e.key)}))}),[E,v,S]),_=(0,s.useRef)({navigate(e){let{itemUrl:t}=e;(0,se.F)(a,t)?window.location.href=t:f.push(t)}}).current,w=(0,s.useRef)((e=>l.transformItems?l.transformItems(e):e.map((e=>({...e,url:p(e.url)}))))).current,I=(0,s.useMemo)((()=>e=>s.createElement(Pe,(0,i.Z)({},e,{onClose:M}))),[M]),C=(0,s.useCallback)((e=>(e.addAlgoliaAgent("docusaurus",d.docusaurusVersion),e)),[d.docusaurusVersion]);return function(e){var t=e.isOpen,n=e.onOpen,r=e.onClose,o=e.onInput,a=e.searchButtonRef;s.useEffect((function(){function e(e){(27===e.keyCode&&t||"k"===e.key.toLowerCase()&&(e.metaKey||e.ctrlKey)||!function(e){var t=e.target,n=t.tagName;return t.isContentEditable||"INPUT"===n||"SELECT"===n||"TEXTAREA"===n}(e)&&"/"===e.key&&!t)&&(e.preventDefault(),t?r():document.body.classList.contains("DocSearch--active")||document.body.classList.contains("DocSearch--active")||n()),a&&a.current===document.activeElement&&o&&/[a-zA-Z0-9]/.test(String.fromCharCode(e.keyCode))&&o(e)}return window.addEventListener("keydown",e),function(){window.removeEventListener("keydown",e)}}),[t,n,r,o,a])}({isOpen:x,onOpen:L,onClose:M,onInput:k,searchButtonRef:y}),s.createElement(s.Fragment,null,s.createElement(we.Z,null,s.createElement("link",{rel:"preconnect",href:"https://"+l.appId+"-dsn.algolia.net",crossOrigin:"anonymous"})),s.createElement(_e,{onTouchStart:E,onFocus:E,onMouseOver:E,onClick:L,ref:y,translations:De.button}),x&&Te&&b.current&&(0,Re.createPortal)(s.createElement(Te,(0,i.Z)({onClose:M,initialScrollY:window.scrollY,initialQuery:A,navigator:_,transformItems:w,hitComponent:Oe,transformSearchClient:C},l.searchPagePath&&{resultsFooterComponent:I},l,{searchParameters:g,placeholder:De.placeholder,translations:De.modal})),b.current))}function Ne(){const{siteConfig:e}=(0,pe.Z)();return s.createElement(Fe,e.themeConfig.algolia)}const Be="searchBox_ZlJk";function Ve(e){let{children:t,className:n}=e;return s.createElement("div",{className:(0,r.Z)(n,Be)},t)}var Ge=n(80143),We=n(53438);var Ue=n(60373);const ze=e=>e.docs.find((t=>t.id===e.mainDocId));const He={default:ce,localeDropdown:function(e){let{mobile:t,dropdownItemsBefore:n,dropdownItemsAfter:r,...o}=e;const{i18n:{currentLocale:a,locales:d,localeConfigs:p}}=(0,pe.Z)(),u=(0,fe.l)(),{search:m,hash:h}=(0,c.TH)(),g=[...n,...d.map((e=>{const n=""+("pathname://"+u.createUrl({locale:e,fullyQualified:!1}))+m+h;return{label:p[e].label,lang:p[e].htmlLang,to:n,target:"_self",autoAddBaseUrl:!1,className:e===a?t?"menu__link--active":"dropdown__link--active":""}})),...r],f=t?(0,l.I)({message:"Languages",id:"theme.navbar.mobileLanguageDropdown.label",description:"The label for the mobile language switcher dropdown"}):p[a].label;return s.createElement(ge,(0,i.Z)({},o,{mobile:t,label:s.createElement(s.Fragment,null,s.createElement(be,{className:ye}),f),items:g}))},search:function(e){let{mobile:t,className:n}=e;return t?null:s.createElement(Ve,{className:n},s.createElement(Ne,null))},dropdown:ge,html:function(e){let{value:t,className:n,mobile:o=!1,isDropdownItem:a=!1}=e;const i=a?"li":"div";return s.createElement(i,{className:(0,r.Z)({navbar__item:!o&&!a,"menu__list-item":o},n),dangerouslySetInnerHTML:{__html:t}})},doc:function(e){let{docId:t,label:n,docsPluginId:r,...o}=e;const{activeDoc:a}=(0,Ge.Iw)(r),c=(0,We.vY)(t,r);return null===c?null:s.createElement(ce,(0,i.Z)({exact:!0},o,{isActive:()=>(null==a?void 0:a.path)===c.path||!(null==a||!a.sidebar)&&a.sidebar===c.sidebar,label:null!=n?n:c.id,to:c.path}))},docSidebar:function(e){let{sidebarId:t,label:n,docsPluginId:r,...o}=e;const{activeDoc:a}=(0,Ge.Iw)(r),c=(0,We.oz)(t,r).link;if(!c)throw new Error('DocSidebarNavbarItem: Sidebar with ID "'+t+"\" doesn't have anything to be linked to.");return s.createElement(ce,(0,i.Z)({exact:!0},o,{isActive:()=>(null==a?void 0:a.sidebar)===t,label:null!=n?n:c.label,to:c.path}))},docsVersion:function(e){let{label:t,to:n,docsPluginId:r,...o}=e;const a=(0,We.lO)(r)[0],c=null!=t?t:a.label,l=null!=n?n:(e=>e.docs.find((t=>t.id===e.mainDocId)))(a).path;return s.createElement(ce,(0,i.Z)({},o,{label:c,to:l}))},docsVersionDropdown:function(e){let{mobile:t,docsPluginId:n,dropdownActiveClassDisabled:r,dropdownItemsBefore:o,dropdownItemsAfter:a,...d}=e;const{search:p,hash:u}=(0,c.TH)(),m=(0,Ge.Iw)(n),h=(0,Ge.gB)(n),{savePreferredVersionName:g}=(0,Ue.J)(n),f=[...o,...h.map((e=>{var t;const n=null!=(t=m.alternateDocVersions[e.name])?t:ze(e);return{label:e.label,to:""+n.path+p+u,isActive:()=>e===m.activeVersion,onClick:()=>g(e.name)}})),...a],b=(0,We.lO)(n)[0],y=t&&f.length>1?(0,l.I)({id:"theme.navbar.mobileVersionsDropdown.label",message:"Versions",description:"The label for the navbar versions dropdown on mobile view"}):b.label,x=t&&f.length>1?void 0:ze(b).path;return f.length<=1?s.createElement(ce,(0,i.Z)({},d,{mobile:t,label:y,to:x,isActive:r?()=>!1:void 0})):s.createElement(ge,(0,i.Z)({},d,{mobile:t,label:y,to:x,items:f,isActive:r?()=>!1:void 0}))}};function je(e){let{type:t,...n}=e;const r=function(e,t){return e&&"default"!==e?e:"items"in t?"dropdown":"default"}(t,n),o=He[r];if(!o)throw new Error('No NavbarItem component found for type "'+t+'".');return s.createElement(o,n)}function $e(){const e=(0,R.e)(),t=(0,v.L)().navbar.items;return s.createElement("ul",{className:"menu__list"},t.map(((t,n)=>s.createElement(je,(0,i.Z)({mobile:!0},t,{onClick:()=>e.toggle(),key:n})))))}function Ze(e){return s.createElement("button",(0,i.Z)({},e,{type:"button",className:"clean-btn navbar-sidebar__back"}),s.createElement(l.Z,{id:"theme.navbar.mobileSidebarSecondaryMenu.backButtonLabel",description:"The label of the back button to return to main menu, inside the mobile navbar sidebar secondary menu (notably used to display the docs sidebar)"},"\u2190 Back to main menu"))}function Ke(){const e=0===(0,v.L)().navbar.items.length,t=B();return s.createElement(s.Fragment,null,!e&&s.createElement(Ze,{onClick:()=>t.hide()}),t.content)}function Xe(){const e=(0,R.e)();var t;return void 0===(t=e.shown)&&(t=!0),(0,s.useEffect)((()=>(document.body.style.overflow=t?"hidden":"visible",()=>{document.body.style.overflow="visible"})),[t]),e.shouldRender?s.createElement(V,{header:s.createElement(J,null),primaryMenu:s.createElement($e,null),secondaryMenu:s.createElement(Ke,null)}):null}const Ye="navbarHideable_m1mJ",qe="navbarHidden_jGov";function Je(e){return s.createElement("div",(0,i.Z)({role:"presentation"},e,{className:(0,r.Z)("navbar-sidebar__backdrop",e.className)}))}function et(e){let{children:t}=e;const{navbar:{hideOnScroll:n,style:o}}=(0,v.L)(),a=(0,R.e)(),{navbarRef:i,isNavbarVisible:c}=function(e){const[t,n]=(0,s.useState)(e),r=(0,s.useRef)(!1),o=(0,s.useRef)(0),a=(0,s.useCallback)((e=>{null!==e&&(o.current=e.getBoundingClientRect().height)}),[]);return(0,D.RF)(((t,s)=>{let{scrollY:a}=t;if(!e)return;if(a<o.current)return void n(!0);if(r.current)return void(r.current=!1);const i=null==s?void 0:s.scrollY,c=document.documentElement.scrollHeight-o.current,l=window.innerHeight;i&&a>=i?n(!1):a+l<c&&n(!0)})),(0,d.S)((t=>{if(!e)return;const s=t.location.hash;if(s?document.getElementById(s.substring(1)):void 0)return r.current=!0,void n(!1);n(!0)})),{navbarRef:a,isNavbarVisible:t}}(n);return s.createElement("nav",{ref:i,"aria-label":(0,l.I)({id:"theme.NavBar.navAriaLabel",message:"Main",description:"The ARIA label for the main navigation"}),className:(0,r.Z)("navbar","navbar--fixed-top",n&&[Ye,!c&&qe],{"navbar--dark":"dark"===o,"navbar--primary":"primary"===o,"navbar-sidebar--show":a.shown})},t,s.createElement(Je,{onClick:a.toggle}),s.createElement(Xe,null))}var tt=n(18780);const nt="errorBoundaryError_a6uf";function st(e){return s.createElement("button",(0,i.Z)({type:"button"},e),s.createElement(l.Z,{id:"theme.ErrorPageContent.tryAgain",description:"The label of the button to try again rendering when the React error boundary captures an error"},"Try again"))}function rt(e){let{error:t}=e;const n=(0,tt.getErrorCausalChain)(t).map((e=>e.message)).join("\n\nCause:\n");return s.createElement("p",{className:nt},n)}class ot extends s.Component{componentDidCatch(e,t){throw this.props.onError(e,t)}render(){return this.props.children}}function at(e){let{width:t=30,height:n=30,className:r,...o}=e;return s.createElement("svg",(0,i.Z)({className:r,width:t,height:n,viewBox:"0 0 30 30","aria-hidden":"true"},o),s.createElement("path",{stroke:"currentColor",strokeLinecap:"round",strokeMiterlimit:"10",strokeWidth:"2",d:"M4 7h22M4 15h22M4 23h22"}))}function it(){const{toggle:e,shown:t}=(0,R.e)();return s.createElement("button",{onClick:e,"aria-label":(0,l.I)({id:"theme.docs.sidebar.toggleSidebarButtonAriaLabel",message:"Toggle navigation bar",description:"The ARIA label for hamburger menu button of mobile navigation"}),"aria-expanded":t,className:"navbar__toggle clean-btn",type:"button"},s.createElement(at,null))}const ct="colorModeToggle_DEke";function lt(e){let{items:t}=e;return s.createElement(s.Fragment,null,t.map(((e,t)=>s.createElement(ot,{key:t,onError:t=>new Error("A theme navbar item failed to render.\nPlease double-check the following navbar item (themeConfig.navbar.items) of your Docusaurus config:\n"+JSON.stringify(e,null,2),{cause:t})},s.createElement(je,e)))))}function dt(e){let{left:t,right:n}=e;return s.createElement("div",{className:"navbar__inner"},s.createElement("div",{className:"navbar__items"},t),s.createElement("div",{className:"navbar__items navbar__items--right"},n))}function pt(){const e=(0,R.e)(),t=(0,v.L)().navbar.items,[n,r]=function(e){function t(e){var t;return"left"===(null!=(t=e.position)?t:"right")}return[e.filter(t),e.filter((e=>!t(e)))]}(t),o=t.find((e=>"search"===e.type));return s.createElement(dt,{left:s.createElement(s.Fragment,null,!e.disabled&&s.createElement(it,null),s.createElement(Y,null),s.createElement(lt,{items:n})),right:s.createElement(s.Fragment,null,s.createElement(lt,{items:r}),s.createElement(K,{className:ct}),!o&&s.createElement(Ve,null,s.createElement(Ne,null)))})}function ut(){return s.createElement(et,null,s.createElement(pt,null))}function mt(e){let{item:t}=e;const{to:n,href:r,label:o,prependBaseUrlToHref:a,...c}=t,l=(0,te.Z)(n),d=(0,te.Z)(r,{forcePrependBaseUrl:!0});return s.createElement(ee.Z,(0,i.Z)({className:"footer__link-item"},r?{href:a?d:r}:{to:l},c),o,r&&!(0,ne.Z)(r)&&s.createElement(re.Z,null))}function ht(e){var t;let{item:n}=e;return n.html?s.createElement("li",{className:"footer__item",dangerouslySetInnerHTML:{__html:n.html}}):s.createElement("li",{key:null!=(t=n.href)?t:n.to,className:"footer__item"},s.createElement(mt,{item:n}))}function gt(e){let{column:t}=e;return s.createElement("div",{className:"col footer__col"},s.createElement("div",{className:"footer__title"},t.title),s.createElement("ul",{className:"footer__items clean-list"},t.items.map(((e,t)=>s.createElement(ht,{key:t,item:e})))))}function ft(e){let{columns:t}=e;return s.createElement("div",{className:"row footer__links"},t.map(((e,t)=>s.createElement(gt,{key:t,column:e}))))}function bt(){return s.createElement("span",{className:"footer__link-separator"},"\xb7")}function yt(e){let{item:t}=e;return t.html?s.createElement("span",{className:"footer__link-item",dangerouslySetInnerHTML:{__html:t.html}}):s.createElement(mt,{item:t})}function xt(e){let{links:t}=e;return s.createElement("div",{className:"footer__links text--center"},s.createElement("div",{className:"footer__links"},t.map(((e,n)=>s.createElement(s.Fragment,{key:n},s.createElement(yt,{item:e}),t.length!==n+1&&s.createElement(bt,null))))))}function vt(e){let{links:t}=e;return function(e){return"title"in e[0]}(t)?s.createElement(ft,{columns:t}):s.createElement(xt,{links:t})}var At=n(50941);const St="footerLogoLink_BH7S";function Et(e){var t;let{logo:n}=e;const{withBaseUrl:o}=(0,te.C)(),a={light:o(n.src),dark:o(null!=(t=n.srcDark)?t:n.src)};return s.createElement(At.Z,{className:(0,r.Z)("footer__logo",n.className),alt:n.alt,sources:a,width:n.width,height:n.height,style:n.style})}function Lt(e){let{logo:t}=e;return t.href?s.createElement(ee.Z,{href:t.href,className:St,target:t.target},s.createElement(Et,{logo:t})):s.createElement(Et,{logo:t})}function Mt(e){let{copyright:t}=e;return s.createElement("div",{className:"footer__copyright",dangerouslySetInnerHTML:{__html:t}})}function kt(e){let{style:t,links:n,logo:o,copyright:a}=e;return s.createElement("footer",{className:(0,r.Z)("footer",{"footer--dark":"dark"===t})},s.createElement("div",{className:"container container-fluid"},n,(o||a)&&s.createElement("div",{className:"footer__bottom text--center"},o&&s.createElement("div",{className:"margin-bottom--sm"},o),a)))}function _t(){const{footer:e}=(0,v.L)();if(!e)return null;const{copyright:t,links:n,logo:r,style:o}=e;return s.createElement(kt,{style:o,links:n&&n.length>0&&s.createElement(vt,{links:n}),logo:r&&s.createElement(Lt,{logo:r}),copyright:t&&s.createElement(Mt,{copyright:t})})}const wt=s.memo(_t),It=(0,T.Qc)([G.S,A.pl,D.OC,Ue.L5,a.VC,function(e){let{children:t}=e;return s.createElement(O.n2,null,s.createElement(R.M,null,s.createElement(F,null,t)))}]);function Ct(e){let{children:t}=e;return s.createElement(It,null,t)}function Qt(e){let{error:t,tryAgain:n}=e;return s.createElement("main",{className:"container margin-vert--xl"},s.createElement("div",{className:"row"},s.createElement("div",{className:"col col--6 col--offset-3"},s.createElement("h1",{className:"hero__title"},s.createElement(l.Z,{id:"theme.ErrorPageContent.title",description:"The title of the fallback page when the page crashed"},"This page crashed.")),s.createElement("div",{className:"margin-vert--lg"},s.createElement(st,{onClick:n,className:"button button--primary shadow--lw"})),s.createElement("hr",null),s.createElement("div",{className:"margin-vert--md"},s.createElement(rt,{error:t})))))}const Rt="mainWrapper_z2l0";function Dt(e){const{children:t,noFooter:n,wrapperClassName:i,title:c,description:l}=e;return(0,b.t)(),s.createElement(Ct,null,s.createElement(a.d,{title:c,description:l}),s.createElement(x,null),s.createElement(Q,null),s.createElement(ut,null),s.createElement("div",{id:p,className:(0,r.Z)(f.k.wrapper.main,Rt,i)},s.createElement(o.Z,{fallback:e=>s.createElement(Qt,e)},t)),!n&&s.createElement(wt,null))}},21327:(e,t,n)=>{"use strict";n.d(t,{Z:()=>p});var s=n(83117),r=n(67294),o=n(39960),a=n(44996),i=n(52263),c=n(86668),l=n(50941);function d(e){let{logo:t,alt:n,imageClassName:s}=e;const o={light:(0,a.Z)(t.src),dark:(0,a.Z)(t.srcDark||t.src)},i=r.createElement(l.Z,{className:t.className,sources:o,height:t.height,width:t.width,alt:n,style:t.style});return s?r.createElement("div",{className:s},i):i}function p(e){var t;const{siteConfig:{title:n}}=(0,i.Z)(),{navbar:{title:l,logo:p}}=(0,c.L)(),{imageClassName:u,titleClassName:m,...h}=e,g=(0,a.Z)((null==p?void 0:p.href)||"/"),f=l?"":n,b=null!=(t=null==p?void 0:p.alt)?t:f;return r.createElement(o.Z,(0,s.Z)({to:g},h,(null==p?void 0:p.target)&&{target:p.target}),p&&r.createElement(d,{logo:p,alt:b,imageClassName:u}),null!=l&&r.createElement("b",{className:m},l))}},90197:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(35742);function o(e){let{locale:t,version:n,tag:o}=e;const a=t;return s.createElement(r.Z,null,t&&s.createElement("meta",{name:"docusaurus_locale",content:t}),n&&s.createElement("meta",{name:"docusaurus_version",content:n}),o&&s.createElement("meta",{name:"docusaurus_tag",content:o}),a&&s.createElement("meta",{name:"docsearch:language",content:a}),n&&s.createElement("meta",{name:"docsearch:version",content:n}),o&&s.createElement("meta",{name:"docsearch:docusaurus_tag",content:o}))}},50941:(e,t,n)=>{"use strict";n.d(t,{Z:()=>l});var s=n(83117),r=n(67294),o=n(86010),a=n(72389),i=n(92949);const c={themedImage:"themedImage_ToTc","themedImage--light":"themedImage--light_HNdA","themedImage--dark":"themedImage--dark_i4oU"};function l(e){const t=(0,a.Z)(),{colorMode:n}=(0,i.I)(),{sources:l,className:d,alt:p,...u}=e,m=t?"dark"===n?["dark"]:["light"]:["light","dark"];return r.createElement(r.Fragment,null,m.map((e=>r.createElement("img",(0,s.Z)({key:e,src:l[e],alt:p,className:(0,o.Z)(c.themedImage,c["themedImage--"+e],d)},u)))))}},86043:(e,t,n)=>{"use strict";n.d(t,{u:()=>i,z:()=>g});var s=n(83117),r=n(67294),o=n(10412),a=n(91442);function i(e){let{initialState:t}=e;const[n,s]=(0,r.useState)(null!=t&&t),o=(0,r.useCallback)((()=>{s((e=>!e))}),[]);return{collapsed:n,setCollapsed:s,toggleCollapsed:o}}const c={display:"none",overflow:"hidden",height:"0px"},l={display:"block",overflow:"visible",height:"auto"};function d(e,t){const n=t?c:l;e.style.display=n.display,e.style.overflow=n.overflow,e.style.height=n.height}function p(e){let{collapsibleRef:t,collapsed:n,animation:s}=e;const o=(0,r.useRef)(!1);(0,r.useEffect)((()=>{const e=t.current;function r(){var t,n;const r=e.scrollHeight,o=null!=(t=null==s?void 0:s.duration)?t:function(e){if((0,a.n)())return 1;const t=e/36;return Math.round(10*(4+15*t**.25+t/5))}(r);return{transition:"height "+o+"ms "+(null!=(n=null==s?void 0:s.easing)?n:"ease-in-out"),height:r+"px"}}function i(){const t=r();e.style.transition=t.transition,e.style.height=t.height}if(!o.current)return d(e,n),void(o.current=!0);return e.style.willChange="height",function(){const t=requestAnimationFrame((()=>{n?(i(),requestAnimationFrame((()=>{e.style.height=c.height,e.style.overflow=c.overflow}))):(e.style.display="block",requestAnimationFrame((()=>{i()})))}));return()=>cancelAnimationFrame(t)}()}),[t,n,s])}function u(e){if(!o.Z.canUseDOM)return e?c:l}function m(e){let{as:t="div",collapsed:n,children:s,animation:o,onCollapseTransitionEnd:a,className:i,disableSSRStyle:c}=e;const l=(0,r.useRef)(null);return p({collapsibleRef:l,collapsed:n,animation:o}),r.createElement(t,{ref:l,style:c?void 0:u(n),onTransitionEnd:e=>{"height"===e.propertyName&&(d(l.current,n),null==a||a(n))},className:i},s)}function h(e){let{collapsed:t,...n}=e;const[o,a]=(0,r.useState)(!t),[i,c]=(0,r.useState)(t);return(0,r.useLayoutEffect)((()=>{t||a(!0)}),[t]),(0,r.useLayoutEffect)((()=>{o&&c(t)}),[o,t]),o?r.createElement(m,(0,s.Z)({},n,{collapsed:i})):null}function g(e){let{lazy:t,...n}=e;const s=t?h:m;return r.createElement(s,n)}},59689:(e,t,n)=>{"use strict";n.d(t,{nT:()=>h,pl:()=>m});var s=n(67294),r=n(72389),o=n(50012),a=n(902),i=n(86668);const c=(0,o.WA)("docusaurus.announcement.dismiss"),l=(0,o.WA)("docusaurus.announcement.id"),d=()=>"true"===c.get(),p=e=>c.set(String(e)),u=s.createContext(null);function m(e){let{children:t}=e;const n=function(){const{announcementBar:e}=(0,i.L)(),t=(0,r.Z)(),[n,o]=(0,s.useState)((()=>!!t&&d()));(0,s.useEffect)((()=>{o(d())}),[]);const a=(0,s.useCallback)((()=>{p(!0),o(!0)}),[]);return(0,s.useEffect)((()=>{if(!e)return;const{id:t}=e;let n=l.get();"annoucement-bar"===n&&(n="announcement-bar");const s=t!==n;l.set(t),s&&p(!1),!s&&d()||o(!1)}),[e]),(0,s.useMemo)((()=>({isActive:!!e&&!n,close:a})),[e,n,a])}();return s.createElement(u.Provider,{value:n},t)}function h(){const e=(0,s.useContext)(u);if(!e)throw new a.i6("AnnouncementBarProvider");return e}},92949:(e,t,n)=>{"use strict";n.d(t,{I:()=>f,S:()=>g});var s=n(67294),r=n(10412),o=n(902),a=n(50012),i=n(86668);const c=s.createContext(void 0),l="theme",d=(0,a.WA)(l),p="light",u="dark",m=e=>e===u?u:p;function h(){const{colorMode:{defaultMode:e,disableSwitch:t,respectPrefersColorScheme:n}}=(0,i.L)(),[o,a]=(0,s.useState)((e=>r.Z.canUseDOM?m(document.documentElement.getAttribute("data-theme")):m(e))(e));(0,s.useEffect)((()=>{t&&d.del()}),[t]);const c=(0,s.useCallback)((function(t,s){void 0===s&&(s={});const{persist:r=!0}=s;t?(a(t),r&&(e=>{d.set(m(e))})(t)):(a(n?window.matchMedia("(prefers-color-scheme: dark)").matches?u:p:e),d.del())}),[n,e]);(0,s.useEffect)((()=>{document.documentElement.setAttribute("data-theme",m(o))}),[o]),(0,s.useEffect)((()=>{if(t)return;const e=e=>{if(e.key!==l)return;const t=d.get();null!==t&&c(m(t))};return window.addEventListener("storage",e),()=>window.removeEventListener("storage",e)}),[t,c]);const h=(0,s.useRef)(!1);return(0,s.useEffect)((()=>{if(t&&!n)return;const e=window.matchMedia("(prefers-color-scheme: dark)"),s=()=>{window.matchMedia("print").matches||h.current?h.current=window.matchMedia("print").matches:c(null)};return e.addListener(s),()=>e.removeListener(s)}),[c,t,n]),(0,s.useMemo)((()=>({colorMode:o,setColorMode:c,get isDarkTheme(){return o===u},setLightTheme(){c(p)},setDarkTheme(){c(u)}})),[o,c])}function g(e){let{children:t}=e;const n=h();return s.createElement(c.Provider,{value:n},t)}function f(){const e=(0,s.useContext)(c);if(null==e)throw new o.i6("ColorModeProvider","Please see https://docusaurus.io/docs/api/themes/configuration#use-color-mode.");return e}},60373:(e,t,n)=>{"use strict";n.d(t,{J:()=>x,L5:()=>b,Oh:()=>v});var s=n(67294),r=n(80143),o=n(29935),a=n(86668),i=n(53438),c=n(902),l=n(50012);const d=e=>"docs-preferred-version-"+e,p=(e,t,n)=>{(0,l.WA)(d(e),{persistence:t}).set(n)},u=(e,t)=>(0,l.WA)(d(e),{persistence:t}).get(),m=(e,t)=>{(0,l.WA)(d(e),{persistence:t}).del()};const h=s.createContext(null);function g(){const e=(0,r._r)(),t=(0,a.L)().docs.versionPersistence,n=(0,s.useMemo)((()=>Object.keys(e)),[e]),[o,i]=(0,s.useState)((()=>(e=>Object.fromEntries(e.map((e=>[e,{preferredVersionName:null}]))))(n)));(0,s.useEffect)((()=>{i(function(e){let{pluginIds:t,versionPersistence:n,allDocsData:s}=e;function r(e){const t=u(e,n);return s[e].versions.some((e=>e.name===t))?{preferredVersionName:t}:(m(e,n),{preferredVersionName:null})}return Object.fromEntries(t.map((e=>[e,r(e)])))}({allDocsData:e,versionPersistence:t,pluginIds:n}))}),[e,t,n]);return[o,(0,s.useMemo)((()=>({savePreferredVersion:function(e,n){p(e,t,n),i((t=>({...t,[e]:{preferredVersionName:n}})))}})),[t])]}function f(e){let{children:t}=e;const n=g();return s.createElement(h.Provider,{value:n},t)}function b(e){let{children:t}=e;return i.cE?s.createElement(f,null,t):s.createElement(s.Fragment,null,t)}function y(){const e=(0,s.useContext)(h);if(!e)throw new c.i6("DocsPreferredVersionContextProvider");return e}function x(e){var t;void 0===e&&(e=o.m);const n=(0,r.zh)(e),[a,i]=y(),{preferredVersionName:c}=a[e];return{preferredVersion:null!=(t=n.versions.find((e=>e.name===c)))?t:null,savePreferredVersionName:(0,s.useCallback)((t=>{i.savePreferredVersion(e,t)}),[i,e])}}function v(){const e=(0,r._r)(),[t]=y();function n(n){var s;const r=e[n],{preferredVersionName:o}=t[n];return null!=(s=r.versions.find((e=>e.name===o)))?s:null}const s=Object.keys(e);return Object.fromEntries(s.map((e=>[e,n(e)])))}},1116:(e,t,n)=>{"use strict";n.d(t,{V:()=>c,b:()=>i});var s=n(67294),r=n(902);const o=Symbol("EmptyContext"),a=s.createContext(o);function i(e){let{children:t,name:n,items:r}=e;const o=(0,s.useMemo)((()=>n&&r?{name:n,items:r}:null),[n,r]);return s.createElement(a.Provider,{value:o},t)}function c(){const e=(0,s.useContext)(a);if(e===o)throw new r.i6("DocsSidebarProvider");return e}},93163:(e,t,n)=>{"use strict";n.d(t,{M:()=>p,e:()=>u});var s=n(67294),r=n(13102),o=n(87524),a=n(91980),i=n(86668),c=n(902);const l=s.createContext(void 0);function d(){const e=function(){const e=(0,r.HY)(),{items:t}=(0,i.L)().navbar;return 0===t.length&&!e.component}(),t=(0,o.i)(),n=!e&&"mobile"===t,[c,l]=(0,s.useState)(!1);(0,a.Rb)((()=>{if(c)return l(!1),!1}));const d=(0,s.useCallback)((()=>{l((e=>!e))}),[]);return(0,s.useEffect)((()=>{"desktop"===t&&l(!1)}),[t]),(0,s.useMemo)((()=>({disabled:e,shouldRender:n,toggle:d,shown:c})),[e,n,d,c])}function p(e){let{children:t}=e;const n=d();return s.createElement(l.Provider,{value:n},t)}function u(){const e=s.useContext(l);if(void 0===e)throw new c.i6("NavbarMobileSidebarProvider");return e}},13102:(e,t,n)=>{"use strict";n.d(t,{HY:()=>i,Zo:()=>c,n2:()=>a});var s=n(67294),r=n(902);const o=s.createContext(null);function a(e){let{children:t}=e;const n=(0,s.useState)({component:null,props:null});return s.createElement(o.Provider,{value:n},t)}function i(){const e=(0,s.useContext)(o);if(!e)throw new r.i6("NavbarSecondaryMenuContentProvider");return e[0]}function c(e){let{component:t,props:n}=e;const a=(0,s.useContext)(o);if(!a)throw new r.i6("NavbarSecondaryMenuContentProvider");const[,i]=a,c=(0,r.Ql)(n);return(0,s.useEffect)((()=>{i({component:t,props:c})}),[i,t,c]),(0,s.useEffect)((()=>()=>i({component:null,props:null})),[i]),null}},19727:(e,t,n)=>{"use strict";n.d(t,{h:()=>r,t:()=>o});var s=n(67294);const r="navigation-with-keyboard";function o(){(0,s.useEffect)((()=>{function e(e){"keydown"===e.type&&"Tab"===e.key&&document.body.classList.add(r),"mousedown"===e.type&&document.body.classList.remove(r)}return document.addEventListener("keydown",e),document.addEventListener("mousedown",e),()=>{document.body.classList.remove(r),document.removeEventListener("keydown",e),document.removeEventListener("mousedown",e)}}),[])}},66177:(e,t,n)=>{"use strict";n.d(t,{K:()=>a,M:()=>i});var s=n(67294),r=n(52263),o=n(91980);function a(){return(0,o.Nc)("q")}function i(){const{siteConfig:{baseUrl:e,themeConfig:t}}=(0,r.Z)(),{algolia:{searchPagePath:n}}=t;return(0,s.useCallback)((t=>""+e+n+"?q="+encodeURIComponent(t)),[e,n])}},87524:(e,t,n)=>{"use strict";n.d(t,{i:()=>l});var s=n(67294),r=n(10412);const o="desktop",a="mobile",i="ssr";function c(){return r.Z.canUseDOM?window.innerWidth>996?o:a:i}function l(){const[e,t]=(0,s.useState)((()=>c()));return(0,s.useEffect)((()=>{function e(){t(c())}return window.addEventListener("resize",e),()=>{window.removeEventListener("resize",e),clearTimeout(undefined)}}),[]),e}},35281:(e,t,n)=>{"use strict";n.d(t,{k:()=>s});const s={page:{blogListPage:"blog-list-page",blogPostPage:"blog-post-page",blogTagsListPage:"blog-tags-list-page",blogTagPostListPage:"blog-tags-post-list-page",docsDocPage:"docs-doc-page",docsTagsListPage:"docs-tags-list-page",docsTagDocListPage:"docs-tags-doc-list-page",mdxPage:"mdx-page"},wrapper:{main:"main-wrapper",blogPages:"blog-wrapper",docsPages:"docs-wrapper",mdxPages:"mdx-wrapper"},common:{editThisPage:"theme-edit-this-page",lastUpdated:"theme-last-updated",backToTopButton:"theme-back-to-top-button",codeBlock:"theme-code-block",admonition:"theme-admonition",admonitionType:e=>"theme-admonition-"+e},layout:{},docs:{docVersionBanner:"theme-doc-version-banner",docVersionBadge:"theme-doc-version-badge",docBreadcrumbs:"theme-doc-breadcrumbs",docMarkdown:"theme-doc-markdown",docTocMobile:"theme-doc-toc-mobile",docTocDesktop:"theme-doc-toc-desktop",docFooter:"theme-doc-footer",docFooterTagsRow:"theme-doc-footer-tags-row",docFooterEditMetaRow:"theme-doc-footer-edit-meta-row",docSidebarContainer:"theme-doc-sidebar-container",docSidebarMenu:"theme-doc-sidebar-menu",docSidebarItemCategory:"theme-doc-sidebar-item-category",docSidebarItemLink:"theme-doc-sidebar-item-link",docSidebarItemCategoryLevel:e=>"theme-doc-sidebar-item-category-level-"+e,docSidebarItemLinkLevel:e=>"theme-doc-sidebar-item-link-level-"+e},blog:{}}},91442:(e,t,n)=>{"use strict";function s(){return window.matchMedia("(prefers-reduced-motion: reduce)").matches}n.d(t,{n:()=>s})},53438:(e,t,n)=>{"use strict";n.d(t,{Wl:()=>u,_F:()=>h,cE:()=>p,hI:()=>v,lO:()=>b,oz:()=>y,s1:()=>f,vY:()=>x});var s=n(67294),r=n(16550),o=n(18790),a=n(80143),i=n(60373),c=n(1116),l=n(67392),d=n(48596);const p=!!a._r;function u(e){if(e.href)return e.href;for(const t of e.items){if("link"===t.type)return t.href;if("category"===t.type){const e=u(t);if(e)return e}}}const m=(e,t)=>void 0!==e&&(0,d.Mg)(e,t);function h(e,t){return"link"===e.type?m(e.href,t):"category"===e.type&&(m(e.href,t)||((e,t)=>e.some((e=>h(e,t))))(e.items,t))}function g(e){let{sidebarItems:t,pathname:n,onlyCategories:s=!1}=e;const r=[];return function e(t){for(const o of t)if("category"===o.type&&((0,d.Mg)(o.href,n)||e(o.items))||"link"===o.type&&(0,d.Mg)(o.href,n)){return s&&"category"!==o.type||r.unshift(o),!0}return!1}(t),r}function f(){var e;const t=(0,c.V)(),{pathname:n}=(0,r.TH)();return!1!==(null==(e=(0,a.gA)())?void 0:e.pluginData.breadcrumbs)&&t?g({sidebarItems:t.items,pathname:n}):null}function b(e){const{activeVersion:t}=(0,a.Iw)(e),{preferredVersion:n}=(0,i.J)(e),r=(0,a.yW)(e);return(0,s.useMemo)((()=>(0,l.j)([t,n,r].filter(Boolean))),[t,n,r])}function y(e,t){const n=b(t);return(0,s.useMemo)((()=>{const t=n.flatMap((e=>e.sidebars?Object.entries(e.sidebars):[])),s=t.find((t=>t[0]===e));if(!s)throw new Error("Can't find any sidebar with id \""+e+'" in version'+(n.length>1?"s":"")+" "+n.map((e=>e.name)).join(", ")+'".\nAvailable sidebar ids are:\n- '+t.map((e=>e[0])).join("\n- "));return s[1]}),[e,n])}function x(e,t){const n=b(t);return(0,s.useMemo)((()=>{const t=n.flatMap((e=>e.docs)),s=t.find((t=>t.id===e));if(!s){if(n.flatMap((e=>e.draftIds)).includes(e))return null;throw new Error("Couldn't find any doc with id \""+e+'" in version'+(n.length>1?"s":"")+' "'+n.map((e=>e.name)).join(", ")+'".\nAvailable doc ids are:\n- '+(0,l.j)(t.map((e=>e.id))).join("\n- "))}return s}),[e,n])}function v(e){let{route:t,versionMetadata:n}=e;const s=(0,r.TH)(),a=t.routes,i=a.find((e=>(0,r.LX)(s.pathname,e)));if(!i)return null;const c=i.sidebar,l=c?n.docsSidebars[c]:void 0;return{docElement:(0,o.H)(a),sidebarName:c,sidebarItems:l}}},82128:(e,t,n)=>{"use strict";n.d(t,{p:()=>r});var s=n(52263);function r(e){const{siteConfig:t}=(0,s.Z)(),{title:n,titleDelimiter:r}=t;return null!=e&&e.trim().length?e.trim()+" "+r+" "+n:n}},91980:(e,t,n)=>{"use strict";n.d(t,{Nc:()=>l,Rb:()=>i,_X:()=>c});var s=n(67294),r=n(16550),o=n(61688),a=n(902);function i(e){!function(e){const t=(0,r.k6)(),n=(0,a.zX)(e);(0,s.useEffect)((()=>t.block(((e,t)=>n(e,t)))),[t,n])}(((t,n)=>{if("POP"===n)return e(t,n)}))}function c(e){return function(e){const t=(0,r.k6)();return(0,o.useSyncExternalStore)(t.listen,(()=>e(t)),(()=>e(t)))}((t=>null===e?null:new URLSearchParams(t.location.search).get(e)))}function l(e){var t;const n=null!=(t=c(e))?t:"",o=function(){const e=(0,r.k6)();return(0,s.useCallback)(((t,n,s)=>{const r=new URLSearchParams(e.location.search);n?r.set(t,n):r.delete(t),(null!=s&&s.push?e.push:e.replace)({search:r.toString()})}),[e])}();return[n,(0,s.useCallback)(((t,n)=>{o(e,t,n)}),[o,e])]}},67392:(e,t,n)=>{"use strict";function s(e,t){return void 0===t&&(t=(e,t)=>e===t),e.filter(((n,s)=>e.findIndex((e=>t(e,n)))!==s))}function r(e){return Array.from(new Set(e))}n.d(t,{j:()=>r,l:()=>s})},10833:(e,t,n)=>{"use strict";n.d(t,{FG:()=>u,d:()=>d,VC:()=>m});var s=n(67294),r=n(86010),o=n(35742),a=n(30226);function i(){const e=s.useContext(a._);if(!e)throw new Error("Unexpected: no Docusaurus route context found");return e}var c=n(44996),l=n(82128);function d(e){let{title:t,description:n,keywords:r,image:a,children:i}=e;const d=(0,l.p)(t),{withBaseUrl:p}=(0,c.C)(),u=a?p(a,{absolute:!0}):void 0;return s.createElement(o.Z,null,t&&s.createElement("title",null,d),t&&s.createElement("meta",{property:"og:title",content:d}),n&&s.createElement("meta",{name:"description",content:n}),n&&s.createElement("meta",{property:"og:description",content:n}),r&&s.createElement("meta",{name:"keywords",content:Array.isArray(r)?r.join(","):r}),u&&s.createElement("meta",{property:"og:image",content:u}),u&&s.createElement("meta",{name:"twitter:image",content:u}),i)}const p=s.createContext(void 0);function u(e){let{className:t,children:n}=e;const a=s.useContext(p),i=(0,r.Z)(a,t);return s.createElement(p.Provider,{value:i},s.createElement(o.Z,null,s.createElement("html",{className:i})),n)}function m(e){let{children:t}=e;const n=i(),o="plugin-"+n.plugin.name.replace(/docusaurus-(?:plugin|theme)-(?:content-)?/gi,"");const a="plugin-id-"+n.plugin.id;return s.createElement(u,{className:(0,r.Z)(o,a)},t)}},902:(e,t,n)=>{"use strict";n.d(t,{D9:()=>a,Qc:()=>l,Ql:()=>c,i6:()=>i,zX:()=>o});var s=n(67294);const r=n(10412).Z.canUseDOM?s.useLayoutEffect:s.useEffect;function o(e){const t=(0,s.useRef)(e);return r((()=>{t.current=e}),[e]),(0,s.useCallback)((function(){return t.current(...arguments)}),[])}function a(e){const t=(0,s.useRef)();return r((()=>{t.current=e})),t.current}class i extends Error{constructor(e,t){var n,s,r,o;super(),this.name="ReactContextError",this.message="Hook "+(null!=(n=null==(s=this.stack)||null==(r=s.split("\n")[1])||null==(o=r.match(/at (?:\w+\.)?(?<name>\w+)/))?void 0:o.groups.name)?n:"")+" is called outside the <"+e+">. "+(null!=t?t:"")}}function c(e){const t=Object.entries(e);return t.sort(((e,t)=>e[0].localeCompare(t[0]))),(0,s.useMemo)((()=>e),t.flat())}function l(e){return t=>{let{children:n}=t;return s.createElement(s.Fragment,null,e.reduceRight(((e,t)=>s.createElement(t,null,e)),n))}}},98022:(e,t,n)=>{"use strict";function s(e,t){return void 0!==e&&void 0!==t&&new RegExp(e,"gi").test(t)}n.d(t,{F:()=>s})},48596:(e,t,n)=>{"use strict";n.d(t,{Mg:()=>a,Ns:()=>i});var s=n(67294),r=n(723),o=n(52263);function a(e,t){const n=e=>{var t;return null==(t=!e||e.endsWith("/")?e:e+"/")?void 0:t.toLowerCase()};return n(e)===n(t)}function i(){const{baseUrl:e}=(0,o.Z)().siteConfig;return(0,s.useMemo)((()=>function(e){let{baseUrl:t,routes:n}=e;function s(e){return e.path===t&&!0===e.exact}function r(e){return e.path===t&&!e.exact}return function e(t){if(0===t.length)return;return t.find(s)||e(t.filter(r).flatMap((e=>{var t;return null!=(t=e.routes)?t:[]})))}(n)}({routes:r.Z,baseUrl:e})),[e])}},12466:(e,t,n)=>{"use strict";n.d(t,{Ct:()=>m,OC:()=>c,RF:()=>p,o5:()=>u});var s=n(67294),r=n(10412),o=n(72389),a=n(902);const i=s.createContext(void 0);function c(e){let{children:t}=e;const n=function(){const e=(0,s.useRef)(!0);return(0,s.useMemo)((()=>({scrollEventsEnabledRef:e,enableScrollEvents:()=>{e.current=!0},disableScrollEvents:()=>{e.current=!1}})),[])}();return s.createElement(i.Provider,{value:n},t)}function l(){const e=(0,s.useContext)(i);if(null==e)throw new a.i6("ScrollControllerProvider");return e}const d=()=>r.Z.canUseDOM?{scrollX:window.pageXOffset,scrollY:window.pageYOffset}:null;function p(e,t){void 0===t&&(t=[]);const{scrollEventsEnabledRef:n}=l(),r=(0,s.useRef)(d()),o=(0,a.zX)(e);(0,s.useEffect)((()=>{const e=()=>{if(!n.current)return;const e=d();o(e,r.current),r.current=e},t={passive:!0};return e(),window.addEventListener("scroll",e,t),()=>window.removeEventListener("scroll",e,t)}),[o,n,...t])}function u(){const e=l(),t=function(){const e=(0,s.useRef)({elem:null,top:0}),t=(0,s.useCallback)((t=>{e.current={elem:t,top:t.getBoundingClientRect().top}}),[]),n=(0,s.useCallback)((()=>{const{current:{elem:t,top:n}}=e;if(!t)return{restored:!1};const s=t.getBoundingClientRect().top-n;return s&&window.scrollBy({left:0,top:s}),e.current={elem:null,top:0},{restored:0!==s}}),[]);return(0,s.useMemo)((()=>({save:t,restore:n})),[n,t])}(),n=(0,s.useRef)(void 0),r=(0,s.useCallback)((s=>{t.save(s),e.disableScrollEvents(),n.current=()=>{const{restored:s}=t.restore();if(n.current=void 0,s){const t=()=>{e.enableScrollEvents(),window.removeEventListener("scroll",t)};window.addEventListener("scroll",t)}else e.enableScrollEvents()}}),[e,t]);return(0,s.useLayoutEffect)((()=>{queueMicrotask((()=>null==n.current?void 0:n.current()))})),{blockElementScrollPositionUntilNextRender:r}}function m(){const e=(0,s.useRef)(null),t=(0,o.Z)()&&"smooth"===getComputedStyle(document.documentElement).scrollBehavior;return{startScroll:n=>{e.current=t?function(e){return window.scrollTo({top:e,behavior:"smooth"}),()=>{}}(n):function(e){let t=null;const n=document.documentElement.scrollTop>e;return function s(){const r=document.documentElement.scrollTop;(n&&r>e||!n&&r<e)&&(t=requestAnimationFrame(s),window.scrollTo(0,Math.floor(.85*(r-e))+e))}(),()=>t&&cancelAnimationFrame(t)}(n)},cancelScroll:()=>null==e.current?void 0:e.current()}}},43320:(e,t,n)=>{"use strict";n.d(t,{HX:()=>a,_q:()=>c,os:()=>i});var s=n(80143),r=n(52263),o=n(60373);const a="default";function i(e,t){return"docs-"+e+"-"+t}function c(){const{i18n:e}=(0,r.Z)(),t=(0,s._r)(),n=(0,s.WS)(),c=(0,o.Oh)();const l=[a,...Object.keys(t).map((function(e){var s;const r=(null==n?void 0:n.activePlugin.pluginId)===e?n.activeVersion:void 0,o=c[e],a=t[e].versions.find((e=>e.isLast));return i(e,(null!=(s=null!=r?r:o)?s:a).name)}))];return{locale:e.currentLocale,tags:l}}},50012:(e,t,n)=>{"use strict";n.d(t,{Nk:()=>p,WA:()=>d});var s=n(67294),r=n(61688);const o="localStorage";function a(e){let{key:t,oldValue:n,newValue:s,storage:r}=e;if(n===s)return;const o=document.createEvent("StorageEvent");o.initStorageEvent("storage",!1,!1,t,n,s,window.location.href,r),window.dispatchEvent(o)}function i(e){if(void 0===e&&(e=o),"undefined"==typeof window)throw new Error("Browser storage is not available on Node.js/Docusaurus SSR process.");if("none"===e)return null;try{return window[e]}catch(n){return t=n,c||(console.warn("Docusaurus browser storage is not available.\nPossible reasons: running Docusaurus in an iframe, in an incognito browser session, or using too strict browser privacy settings.",t),c=!0),null}var t}let c=!1;const l={get:()=>null,set:()=>{},del:()=>{},listen:()=>()=>{}};function d(e,t){if("undefined"==typeof window)return function(e){function t(){throw new Error('Illegal storage API usage for storage key "'+e+'".\nDocusaurus storage APIs are not supposed to be called on the server-rendering process.\nPlease only call storage APIs in effects and event handlers.')}return{get:t,set:t,del:t,listen:t}}(e);const n=i(null==t?void 0:t.persistence);return null===n?l:{get:()=>{try{return n.getItem(e)}catch(t){return console.error("Docusaurus storage error, can't get key="+e,t),null}},set:t=>{try{const s=n.getItem(e);n.setItem(e,t),a({key:e,oldValue:s,newValue:t,storage:n})}catch(s){console.error("Docusaurus storage error, can't set "+e+"="+t,s)}},del:()=>{try{const t=n.getItem(e);n.removeItem(e),a({key:e,oldValue:t,newValue:null,storage:n})}catch(t){console.error("Docusaurus storage error, can't delete key="+e,t)}},listen:t=>{try{const s=s=>{s.storageArea===n&&s.key===e&&t(s)};return window.addEventListener("storage",s),()=>window.removeEventListener("storage",s)}catch(s){return console.error("Docusaurus storage error, can't listen for changes of key="+e,s),()=>{}}}}}function p(e,t){const n=(0,s.useRef)((()=>null===e?l:d(e,t))).current(),o=(0,s.useCallback)((e=>"undefined"==typeof window?()=>{}:n.listen(e)),[n]);return[(0,r.useSyncExternalStore)(o,(()=>"undefined"==typeof window?null:n.get()),(()=>null)),n]}},94711:(e,t,n)=>{"use strict";n.d(t,{l:()=>o});var s=n(52263),r=n(16550);function o(){const{siteConfig:{baseUrl:e,url:t},i18n:{defaultLocale:n,currentLocale:o}}=(0,s.Z)(),{pathname:a}=(0,r.TH)(),i=o===n?e:e.replace("/"+o+"/","/"),c=a.replace(e,"");return{createUrl:function(e){let{locale:s,fullyQualified:r}=e;return""+(r?t:"")+function(e){return e===n?""+i:""+i+e+"/"}(s)+c}}}},85936:(e,t,n)=>{"use strict";n.d(t,{S:()=>a});var s=n(67294),r=n(16550),o=n(902);function a(e){const t=(0,r.TH)(),n=(0,o.D9)(t),a=(0,o.zX)(e);(0,s.useEffect)((()=>{n&&t!==n&&a({location:t,previousLocation:n})}),[a,t,n])}},86668:(e,t,n)=>{"use strict";n.d(t,{L:()=>r});var s=n(52263);function r(){return(0,s.Z)().siteConfig.themeConfig}},6278:(e,t,n)=>{"use strict";n.d(t,{L:()=>r});var s=n(52263);function r(){const{siteConfig:{themeConfig:e}}=(0,s.Z)();return e}},239:(e,t,n)=>{"use strict";n.d(t,{l:()=>i});var s=n(67294),r=n(98022),o=n(44996),a=n(6278);function i(){const{withBaseUrl:e}=(0,o.C)(),{algolia:{externalUrlRegex:t,replaceSearchResultPathname:n}}=(0,a.L)();return(0,s.useCallback)((s=>{const o=new URL(s);if((0,r.F)(t,o.href))return s;const a=""+(o.pathname+o.hash);return e(function(e,t){return t?e.replaceAll(new RegExp(t.from,"g"),t.to):e}(a,n))}),[e,t,n])}},8802:(e,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){const{trailingSlash:n,baseUrl:s}=t;if(e.startsWith("#"))return e;if(void 0===n)return e;const[r]=e.split(/[#?]/),o="/"===r||r===s?r:(a=r,n?function(e){return e.endsWith("/")?e:e+"/"}(a):function(e){return e.endsWith("/")?e.slice(0,-1):e}(a));var a;return e.replace(r,o)}},54143:(e,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getErrorCausalChain=void 0,t.getErrorCausalChain=function e(t){return t.cause?[t,...e(t.cause)]:[t]}},18780:function(e,t,n){"use strict";var s=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.getErrorCausalChain=t.applyTrailingSlash=t.blogPostContainerID=void 0,t.blogPostContainerID="__blog-post-container";var r=n(8802);Object.defineProperty(t,"applyTrailingSlash",{enumerable:!0,get:function(){return s(r).default}});var o=n(54143);Object.defineProperty(t,"getErrorCausalChain",{enumerable:!0,get:function(){return o.getErrorCausalChain}})},86010:(e,t,n)=>{"use strict";function s(e){var t,n,r="";if("string"==typeof e||"number"==typeof e)r+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;t<e.length;t++)e[t]&&(n=s(e[t]))&&(r&&(r+=" "),r+=n);else for(t in e)e[t]&&(r&&(r+=" "),r+=t);return r}n.d(t,{Z:()=>r});const r=function(){for(var e,t,n=0,r="";n<arguments.length;)(e=arguments[n++])&&(t=s(e))&&(r&&(r+=" "),r+=t);return r}},99318:(e,t,n)=>{"use strict";n.d(t,{lX:()=>v,q_:()=>k,ob:()=>m,PP:()=>w,Ep:()=>u});var s=n(83117);function r(e){return"/"===e.charAt(0)}function o(e,t){for(var n=t,s=n+1,r=e.length;s<r;n+=1,s+=1)e[n]=e[s];e.pop()}const a=function(e,t){void 0===t&&(t="");var n,s=e&&e.split("/")||[],a=t&&t.split("/")||[],i=e&&r(e),c=t&&r(t),l=i||c;if(e&&r(e)?a=s:s.length&&(a.pop(),a=a.concat(s)),!a.length)return"/";if(a.length){var d=a[a.length-1];n="."===d||".."===d||""===d}else n=!1;for(var p=0,u=a.length;u>=0;u--){var m=a[u];"."===m?o(a,u):".."===m?(o(a,u),p++):p&&(o(a,u),p--)}if(!l)for(;p--;p)a.unshift("..");!l||""===a[0]||a[0]&&r(a[0])||a.unshift("");var h=a.join("/");return n&&"/"!==h.substr(-1)&&(h+="/"),h};var i=n(2177);function c(e){return"/"===e.charAt(0)?e:"/"+e}function l(e){return"/"===e.charAt(0)?e.substr(1):e}function d(e,t){return function(e,t){return 0===e.toLowerCase().indexOf(t.toLowerCase())&&-1!=="/?#".indexOf(e.charAt(t.length))}(e,t)?e.substr(t.length):e}function p(e){return"/"===e.charAt(e.length-1)?e.slice(0,-1):e}function u(e){var t=e.pathname,n=e.search,s=e.hash,r=t||"/";return n&&"?"!==n&&(r+="?"===n.charAt(0)?n:"?"+n),s&&"#"!==s&&(r+="#"===s.charAt(0)?s:"#"+s),r}function m(e,t,n,r){var o;"string"==typeof e?(o=function(e){var t=e||"/",n="",s="",r=t.indexOf("#");-1!==r&&(s=t.substr(r),t=t.substr(0,r));var o=t.indexOf("?");return-1!==o&&(n=t.substr(o),t=t.substr(0,o)),{pathname:t,search:"?"===n?"":n,hash:"#"===s?"":s}}(e),o.state=t):(void 0===(o=(0,s.Z)({},e)).pathname&&(o.pathname=""),o.search?"?"!==o.search.charAt(0)&&(o.search="?"+o.search):o.search="",o.hash?"#"!==o.hash.charAt(0)&&(o.hash="#"+o.hash):o.hash="",void 0!==t&&void 0===o.state&&(o.state=t));try{o.pathname=decodeURI(o.pathname)}catch(i){throw i instanceof URIError?new URIError('Pathname "'+o.pathname+'" could not be decoded. This is likely caused by an invalid percent-encoding.'):i}return n&&(o.key=n),r?o.pathname?"/"!==o.pathname.charAt(0)&&(o.pathname=a(o.pathname,r.pathname)):o.pathname=r.pathname:o.pathname||(o.pathname="/"),o}function h(){var e=null;var t=[];return{setPrompt:function(t){return e=t,function(){e===t&&(e=null)}},confirmTransitionTo:function(t,n,s,r){if(null!=e){var o="function"==typeof e?e(t,n):e;"string"==typeof o?"function"==typeof s?s(o,r):r(!0):r(!1!==o)}else r(!0)},appendListener:function(e){var n=!0;function s(){n&&e.apply(void 0,arguments)}return t.push(s),function(){n=!1,t=t.filter((function(e){return e!==s}))}},notifyListeners:function(){for(var e=arguments.length,n=new Array(e),s=0;s<e;s++)n[s]=arguments[s];t.forEach((function(e){return e.apply(void 0,n)}))}}}var g=!("undefined"==typeof window||!window.document||!window.document.createElement);function f(e,t){t(window.confirm(e))}var b="popstate",y="hashchange";function x(){try{return window.history.state||{}}catch(e){return{}}}function v(e){void 0===e&&(e={}),g||(0,i.Z)(!1);var t,n=window.history,r=(-1===(t=window.navigator.userAgent).indexOf("Android 2.")&&-1===t.indexOf("Android 4.0")||-1===t.indexOf("Mobile Safari")||-1!==t.indexOf("Chrome")||-1!==t.indexOf("Windows Phone"))&&window.history&&"pushState"in window.history,o=!(-1===window.navigator.userAgent.indexOf("Trident")),a=e,l=a.forceRefresh,v=void 0!==l&&l,A=a.getUserConfirmation,S=void 0===A?f:A,E=a.keyLength,L=void 0===E?6:E,M=e.basename?p(c(e.basename)):"";function k(e){var t=e||{},n=t.key,s=t.state,r=window.location,o=r.pathname+r.search+r.hash;return M&&(o=d(o,M)),m(o,s,n)}function _(){return Math.random().toString(36).substr(2,L)}var w=h();function I(e){(0,s.Z)(G,e),G.length=n.length,w.notifyListeners(G.location,G.action)}function C(e){(function(e){return void 0===e.state&&-1===navigator.userAgent.indexOf("CriOS")})(e)||D(k(e.state))}function Q(){D(k(x()))}var R=!1;function D(e){if(R)R=!1,I();else{w.confirmTransitionTo(e,"POP",S,(function(t){t?I({action:"POP",location:e}):function(e){var t=G.location,n=O.indexOf(t.key);-1===n&&(n=0);var s=O.indexOf(e.key);-1===s&&(s=0);var r=n-s;r&&(R=!0,F(r))}(e)}))}}var T=k(x()),O=[T.key];function P(e){return M+u(e)}function F(e){n.go(e)}var N=0;function B(e){1===(N+=e)&&1===e?(window.addEventListener(b,C),o&&window.addEventListener(y,Q)):0===N&&(window.removeEventListener(b,C),o&&window.removeEventListener(y,Q))}var V=!1;var G={length:n.length,action:"POP",location:T,createHref:P,push:function(e,t){var s="PUSH",o=m(e,t,_(),G.location);w.confirmTransitionTo(o,s,S,(function(e){if(e){var t=P(o),a=o.key,i=o.state;if(r)if(n.pushState({key:a,state:i},null,t),v)window.location.href=t;else{var c=O.indexOf(G.location.key),l=O.slice(0,c+1);l.push(o.key),O=l,I({action:s,location:o})}else window.location.href=t}}))},replace:function(e,t){var s="REPLACE",o=m(e,t,_(),G.location);w.confirmTransitionTo(o,s,S,(function(e){if(e){var t=P(o),a=o.key,i=o.state;if(r)if(n.replaceState({key:a,state:i},null,t),v)window.location.replace(t);else{var c=O.indexOf(G.location.key);-1!==c&&(O[c]=o.key),I({action:s,location:o})}else window.location.replace(t)}}))},go:F,goBack:function(){F(-1)},goForward:function(){F(1)},block:function(e){void 0===e&&(e=!1);var t=w.setPrompt(e);return V||(B(1),V=!0),function(){return V&&(V=!1,B(-1)),t()}},listen:function(e){var t=w.appendListener(e);return B(1),function(){B(-1),t()}}};return G}var A="hashchange",S={hashbang:{encodePath:function(e){return"!"===e.charAt(0)?e:"!/"+l(e)},decodePath:function(e){return"!"===e.charAt(0)?e.substr(1):e}},noslash:{encodePath:l,decodePath:c},slash:{encodePath:c,decodePath:c}};function E(e){var t=e.indexOf("#");return-1===t?e:e.slice(0,t)}function L(){var e=window.location.href,t=e.indexOf("#");return-1===t?"":e.substring(t+1)}function M(e){window.location.replace(E(window.location.href)+"#"+e)}function k(e){void 0===e&&(e={}),g||(0,i.Z)(!1);var t=window.history,n=(window.navigator.userAgent.indexOf("Firefox"),e),r=n.getUserConfirmation,o=void 0===r?f:r,a=n.hashType,l=void 0===a?"slash":a,b=e.basename?p(c(e.basename)):"",y=S[l],x=y.encodePath,v=y.decodePath;function k(){var e=v(L());return b&&(e=d(e,b)),m(e)}var _=h();function w(e){(0,s.Z)(V,e),V.length=t.length,_.notifyListeners(V.location,V.action)}var I=!1,C=null;function Q(){var e,t,n=L(),s=x(n);if(n!==s)M(s);else{var r=k(),a=V.location;if(!I&&(t=r,(e=a).pathname===t.pathname&&e.search===t.search&&e.hash===t.hash))return;if(C===u(r))return;C=null,function(e){if(I)I=!1,w();else{var t="POP";_.confirmTransitionTo(e,t,o,(function(n){n?w({action:t,location:e}):function(e){var t=V.location,n=O.lastIndexOf(u(t));-1===n&&(n=0);var s=O.lastIndexOf(u(e));-1===s&&(s=0);var r=n-s;r&&(I=!0,P(r))}(e)}))}}(r)}}var R=L(),D=x(R);R!==D&&M(D);var T=k(),O=[u(T)];function P(e){t.go(e)}var F=0;function N(e){1===(F+=e)&&1===e?window.addEventListener(A,Q):0===F&&window.removeEventListener(A,Q)}var B=!1;var V={length:t.length,action:"POP",location:T,createHref:function(e){var t=document.querySelector("base"),n="";return t&&t.getAttribute("href")&&(n=E(window.location.href)),n+"#"+x(b+u(e))},push:function(e,t){var n="PUSH",s=m(e,void 0,void 0,V.location);_.confirmTransitionTo(s,n,o,(function(e){if(e){var t=u(s),r=x(b+t);if(L()!==r){C=t,function(e){window.location.hash=e}(r);var o=O.lastIndexOf(u(V.location)),a=O.slice(0,o+1);a.push(t),O=a,w({action:n,location:s})}else w()}}))},replace:function(e,t){var n="REPLACE",s=m(e,void 0,void 0,V.location);_.confirmTransitionTo(s,n,o,(function(e){if(e){var t=u(s),r=x(b+t);L()!==r&&(C=t,M(r));var o=O.indexOf(u(V.location));-1!==o&&(O[o]=t),w({action:n,location:s})}}))},go:P,goBack:function(){P(-1)},goForward:function(){P(1)},block:function(e){void 0===e&&(e=!1);var t=_.setPrompt(e);return B||(N(1),B=!0),function(){return B&&(B=!1,N(-1)),t()}},listen:function(e){var t=_.appendListener(e);return N(1),function(){N(-1),t()}}};return V}function _(e,t,n){return Math.min(Math.max(e,t),n)}function w(e){void 0===e&&(e={});var t=e,n=t.getUserConfirmation,r=t.initialEntries,o=void 0===r?["/"]:r,a=t.initialIndex,i=void 0===a?0:a,c=t.keyLength,l=void 0===c?6:c,d=h();function p(e){(0,s.Z)(v,e),v.length=v.entries.length,d.notifyListeners(v.location,v.action)}function g(){return Math.random().toString(36).substr(2,l)}var f=_(i,0,o.length-1),b=o.map((function(e){return m(e,void 0,"string"==typeof e?g():e.key||g())})),y=u;function x(e){var t=_(v.index+e,0,v.entries.length-1),s=v.entries[t];d.confirmTransitionTo(s,"POP",n,(function(e){e?p({action:"POP",location:s,index:t}):p()}))}var v={length:b.length,action:"POP",location:b[f],index:f,entries:b,createHref:y,push:function(e,t){var s="PUSH",r=m(e,t,g(),v.location);d.confirmTransitionTo(r,s,n,(function(e){if(e){var t=v.index+1,n=v.entries.slice(0);n.length>t?n.splice(t,n.length-t,r):n.push(r),p({action:s,location:r,index:t,entries:n})}}))},replace:function(e,t){var s="REPLACE",r=m(e,t,g(),v.location);d.confirmTransitionTo(r,s,n,(function(e){e&&(v.entries[v.index]=r,p({action:s,location:r}))}))},go:x,goBack:function(){x(-1)},goForward:function(){x(1)},canGo:function(e){var t=v.index+e;return t>=0&&t<v.entries.length},block:function(e){return void 0===e&&(e=!1),d.setPrompt(e)},listen:function(e){return d.appendListener(e)}};return v}},8679:(e,t,n)=>{"use strict";var s=n(59864),r={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},o={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},a={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},i={};function c(e){return s.isMemo(e)?a:i[e.$$typeof]||r}i[s.ForwardRef]={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},i[s.Memo]=a;var l=Object.defineProperty,d=Object.getOwnPropertyNames,p=Object.getOwnPropertySymbols,u=Object.getOwnPropertyDescriptor,m=Object.getPrototypeOf,h=Object.prototype;e.exports=function e(t,n,s){if("string"!=typeof n){if(h){var r=m(n);r&&r!==h&&e(t,r,s)}var a=d(n);p&&(a=a.concat(p(n)));for(var i=c(t),g=c(n),f=0;f<a.length;++f){var b=a[f];if(!(o[b]||s&&s[b]||g&&g[b]||i&&i[b])){var y=u(n,b);try{l(t,b,y)}catch(x){}}}}return t}},41143:e=>{"use strict";e.exports=function(e,t,n,s,r,o,a,i){if(!e){var c;if(void 0===t)c=new Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var l=[n,s,r,o,a,i],d=0;(c=new Error(t.replace(/%s/g,(function(){return l[d++]})))).name="Invariant Violation"}throw c.framesToPop=1,c}}},5826:e=>{e.exports=Array.isArray||function(e){return"[object Array]"==Object.prototype.toString.call(e)}},32497:(e,t,n)=>{"use strict";n.r(t)},52295:(e,t,n)=>{"use strict";n.r(t)},74865:function(e,t,n){var s,r;s=function(){var e,t,n={version:"0.2.0"},s=n.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'<div class="bar" role="bar"><div class="peg"></div></div><div class="spinner" role="spinner"><div class="spinner-icon"></div></div>'};function r(e,t,n){return e<t?t:e>n?n:e}function o(e){return 100*(-1+e)}function a(e,t,n){var r;return(r="translate3d"===s.positionUsing?{transform:"translate3d("+o(e)+"%,0,0)"}:"translate"===s.positionUsing?{transform:"translate("+o(e)+"%,0)"}:{"margin-left":o(e)+"%"}).transition="all "+t+"ms "+n,r}n.configure=function(e){var t,n;for(t in e)void 0!==(n=e[t])&&e.hasOwnProperty(t)&&(s[t]=n);return this},n.status=null,n.set=function(e){var t=n.isStarted();e=r(e,s.minimum,1),n.status=1===e?null:e;var o=n.render(!t),l=o.querySelector(s.barSelector),d=s.speed,p=s.easing;return o.offsetWidth,i((function(t){""===s.positionUsing&&(s.positionUsing=n.getPositioningCSS()),c(l,a(e,d,p)),1===e?(c(o,{transition:"none",opacity:1}),o.offsetWidth,setTimeout((function(){c(o,{transition:"all "+d+"ms linear",opacity:0}),setTimeout((function(){n.remove(),t()}),d)}),d)):setTimeout(t,d)})),this},n.isStarted=function(){return"number"==typeof n.status},n.start=function(){n.status||n.set(0);var e=function(){setTimeout((function(){n.status&&(n.trickle(),e())}),s.trickleSpeed)};return s.trickle&&e(),this},n.done=function(e){return e||n.status?n.inc(.3+.5*Math.random()).set(1):this},n.inc=function(e){var t=n.status;return t?("number"!=typeof e&&(e=(1-t)*r(Math.random()*t,.1,.95)),t=r(t+e,0,.994),n.set(t)):n.start()},n.trickle=function(){return n.inc(Math.random()*s.trickleRate)},e=0,t=0,n.promise=function(s){return s&&"resolved"!==s.state()?(0===t&&n.start(),e++,t++,s.always((function(){0==--t?(e=0,n.done()):n.set((e-t)/e)})),this):this},n.render=function(e){if(n.isRendered())return document.getElementById("nprogress");d(document.documentElement,"nprogress-busy");var t=document.createElement("div");t.id="nprogress",t.innerHTML=s.template;var r,a=t.querySelector(s.barSelector),i=e?"-100":o(n.status||0),l=document.querySelector(s.parent);return c(a,{transition:"all 0 linear",transform:"translate3d("+i+"%,0,0)"}),s.showSpinner||(r=t.querySelector(s.spinnerSelector))&&m(r),l!=document.body&&d(l,"nprogress-custom-parent"),l.appendChild(t),t},n.remove=function(){p(document.documentElement,"nprogress-busy"),p(document.querySelector(s.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&m(e)},n.isRendered=function(){return!!document.getElementById("nprogress")},n.getPositioningCSS=function(){var e=document.body.style,t="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return t+"Perspective"in e?"translate3d":t+"Transform"in e?"translate":"margin"};var i=function(){var e=[];function t(){var n=e.shift();n&&n(t)}return function(n){e.push(n),1==e.length&&t()}}(),c=function(){var e=["Webkit","O","Moz","ms"],t={};function n(e){return e.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,(function(e,t){return t.toUpperCase()}))}function s(t){var n=document.body.style;if(t in n)return t;for(var s,r=e.length,o=t.charAt(0).toUpperCase()+t.slice(1);r--;)if((s=e[r]+o)in n)return s;return t}function r(e){return e=n(e),t[e]||(t[e]=s(e))}function o(e,t,n){t=r(t),e.style[t]=n}return function(e,t){var n,s,r=arguments;if(2==r.length)for(n in t)void 0!==(s=t[n])&&t.hasOwnProperty(n)&&o(e,n,s);else o(e,r[1],r[2])}}();function l(e,t){return("string"==typeof e?e:u(e)).indexOf(" "+t+" ")>=0}function d(e,t){var n=u(e),s=n+t;l(n,t)||(e.className=s.substring(1))}function p(e,t){var n,s=u(e);l(e,t)&&(n=s.replace(" "+t+" "," "),e.className=n.substring(1,n.length-1))}function u(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function m(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return n},void 0===(r="function"==typeof s?s.call(t,n,t,e):s)||(e.exports=r)},27418:e=>{"use strict";var t=Object.getOwnPropertySymbols,n=Object.prototype.hasOwnProperty,s=Object.prototype.propertyIsEnumerable;function r(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map((function(e){return t[e]})).join(""))return!1;var s={};return"abcdefghijklmnopqrst".split("").forEach((function(e){s[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},s)).join("")}catch(r){return!1}}()?Object.assign:function(e,o){for(var a,i,c=r(e),l=1;l<arguments.length;l++){for(var d in a=Object(arguments[l]))n.call(a,d)&&(c[d]=a[d]);if(t){i=t(a);for(var p=0;p<i.length;p++)s.call(a,i[p])&&(c[i[p]]=a[i[p]])}}return c}},87410:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=function(){var e=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,t=0,n={},s={util:{encode:function e(t){return t instanceof r?new r(t.type,e(t.content),t.alias):Array.isArray(t)?t.map(e):t.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++t}),e.__id},clone:function e(t,n){var r,o;switch(n=n||{},s.util.type(t)){case"Object":if(o=s.util.objId(t),n[o])return n[o];for(var a in r={},n[o]=r,t)t.hasOwnProperty(a)&&(r[a]=e(t[a],n));return r;case"Array":return o=s.util.objId(t),n[o]?n[o]:(r=[],n[o]=r,t.forEach((function(t,s){r[s]=e(t,n)})),r);default:return t}},getLanguage:function(t){for(;t;){var n=e.exec(t.className);if(n)return n[1].toLowerCase();t=t.parentElement}return"none"},setLanguage:function(t,n){t.className=t.className.replace(RegExp(e,"gi"),""),t.classList.add("language-"+n)},isActive:function(e,t,n){for(var s="no-"+t;e;){var r=e.classList;if(r.contains(t))return!0;if(r.contains(s))return!1;e=e.parentElement}return!!n}},languages:{plain:n,plaintext:n,text:n,txt:n,extend:function(e,t){var n=s.util.clone(s.languages[e]);for(var r in t)n[r]=t[r];return n},insertBefore:function(e,t,n,r){var o=(r=r||s.languages)[e],a={};for(var i in o)if(o.hasOwnProperty(i)){if(i==t)for(var c in n)n.hasOwnProperty(c)&&(a[c]=n[c]);n.hasOwnProperty(i)||(a[i]=o[i])}var l=r[e];return r[e]=a,s.languages.DFS(s.languages,(function(t,n){n===l&&t!=e&&(this[t]=a)})),a},DFS:function e(t,n,r,o){o=o||{};var a=s.util.objId;for(var i in t)if(t.hasOwnProperty(i)){n.call(t,i,t[i],r||i);var c=t[i],l=s.util.type(c);"Object"!==l||o[a(c)]?"Array"!==l||o[a(c)]||(o[a(c)]=!0,e(c,n,i,o)):(o[a(c)]=!0,e(c,n,null,o))}}},plugins:{},highlight:function(e,t,n){var o={code:e,grammar:t,language:n};return s.hooks.run("before-tokenize",o),o.tokens=s.tokenize(o.code,o.grammar),s.hooks.run("after-tokenize",o),r.stringify(s.util.encode(o.tokens),o.language)},tokenize:function(e,t){var n=t.rest;if(n){for(var s in n)t[s]=n[s];delete t.rest}var r=new i;return c(r,r.head,e),a(e,r,t,r.head,0),function(e){var t=[],n=e.head.next;for(;n!==e.tail;)t.push(n.value),n=n.next;return t}(r)},hooks:{all:{},add:function(e,t){var n=s.hooks.all;n[e]=n[e]||[],n[e].push(t)},run:function(e,t){var n=s.hooks.all[e];if(n&&n.length)for(var r,o=0;r=n[o++];)r(t)}},Token:r};function r(e,t,n,s){this.type=e,this.content=t,this.alias=n,this.length=0|(s||"").length}function o(e,t,n,s){e.lastIndex=t;var r=e.exec(n);if(r&&s&&r[1]){var o=r[1].length;r.index+=o,r[0]=r[0].slice(o)}return r}function a(e,t,n,i,d,p){for(var u in n)if(n.hasOwnProperty(u)&&n[u]){var m=n[u];m=Array.isArray(m)?m:[m];for(var h=0;h<m.length;++h){if(p&&p.cause==u+","+h)return;var g=m[h],f=g.inside,b=!!g.lookbehind,y=!!g.greedy,x=g.alias;if(y&&!g.pattern.global){var v=g.pattern.toString().match(/[imsuy]*$/)[0];g.pattern=RegExp(g.pattern.source,v+"g")}for(var A=g.pattern||g,S=i.next,E=d;S!==t.tail&&!(p&&E>=p.reach);E+=S.value.length,S=S.next){var L=S.value;if(t.length>e.length)return;if(!(L instanceof r)){var M,k=1;if(y){if(!(M=o(A,E,e,b))||M.index>=e.length)break;var _=M.index,w=M.index+M[0].length,I=E;for(I+=S.value.length;_>=I;)I+=(S=S.next).value.length;if(E=I-=S.value.length,S.value instanceof r)continue;for(var C=S;C!==t.tail&&(I<w||"string"==typeof C.value);C=C.next)k++,I+=C.value.length;k--,L=e.slice(E,I),M.index-=E}else if(!(M=o(A,0,L,b)))continue;_=M.index;var Q=M[0],R=L.slice(0,_),D=L.slice(_+Q.length),T=E+L.length;p&&T>p.reach&&(p.reach=T);var O=S.prev;if(R&&(O=c(t,O,R),E+=R.length),l(t,O,k),S=c(t,O,new r(u,f?s.tokenize(Q,f):Q,x,Q)),D&&c(t,S,D),k>1){var P={cause:u+","+h,reach:T};a(e,t,n,S.prev,E,P),p&&P.reach>p.reach&&(p.reach=P.reach)}}}}}}function i(){var e={value:null,prev:null,next:null},t={value:null,prev:e,next:null};e.next=t,this.head=e,this.tail=t,this.length=0}function c(e,t,n){var s=t.next,r={value:n,prev:t,next:s};return t.next=r,s.prev=r,e.length++,r}function l(e,t,n){for(var s=t.next,r=0;r<n&&s!==e.tail;r++)s=s.next;t.next=s,s.prev=t,e.length-=r}return r.stringify=function e(t,n){if("string"==typeof t)return t;if(Array.isArray(t)){var r="";return t.forEach((function(t){r+=e(t,n)})),r}var o={type:t.type,content:e(t.content,n),tag:"span",classes:["token",t.type],attributes:{},language:n},a=t.alias;a&&(Array.isArray(a)?Array.prototype.push.apply(o.classes,a):o.classes.push(a)),s.hooks.run("wrap",o);var i="";for(var c in o.attributes)i+=" "+c+'="'+(o.attributes[c]||"").replace(/"/g,"&quot;")+'"';return"<"+o.tag+' class="'+o.classes.join(" ")+'"'+i+">"+o.content+"</"+o.tag+">"},s}(),r=s;s.default=s,r.languages.markup={comment:{pattern:/<!--(?:(?!<!--)[\s\S])*?-->/,greedy:!0},prolog:{pattern:/<\?[\s\S]+?\?>/,greedy:!0},doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|<!--(?:[^-]|-(?!->))*-->)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(^[^\[]*\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^<!|>$|[[\]]/,"doctype-tag":/^DOCTYPE/i,name:/[^\s<>'"]+/}},cdata:{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,greedy:!0},tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"special-attr":[],"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},r.languages.markup.tag.inside["attr-value"].inside.entity=r.languages.markup.entity,r.languages.markup.doctype.inside["internal-subset"].inside=r.languages.markup,r.hooks.add("wrap",(function(e){"entity"===e.type&&(e.attributes.title=e.content.replace(/&amp;/,"&"))})),Object.defineProperty(r.languages.markup.tag,"addInlined",{value:function(e,t){var n={};n["language-"+t]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:r.languages[t]},n.cdata=/^<!\[CDATA\[|\]\]>$/i;var s={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:n}};s["language-"+t]={pattern:/[\s\S]+/,inside:r.languages[t]};var o={};o[e]={pattern:RegExp(/(<__[^>]*>)(?:<!\[CDATA\[(?:[^\]]|\](?!\]>))*\]\]>|(?!<!\[CDATA\[)[\s\S])*?(?=<\/__>)/.source.replace(/__/g,(function(){return e})),"i"),lookbehind:!0,greedy:!0,inside:s},r.languages.insertBefore("markup","cdata",o)}}),Object.defineProperty(r.languages.markup.tag,"addAttribute",{value:function(e,t){r.languages.markup.tag.inside["special-attr"].push({pattern:RegExp(/(^|["'\s])/.source+"(?:"+e+")"+/\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))/.source,"i"),lookbehind:!0,inside:{"attr-name":/^[^\s=]+/,"attr-value":{pattern:/=[\s\S]+/,inside:{value:{pattern:/(^=\s*(["']|(?!["'])))\S[\s\S]*(?=\2$)/,lookbehind:!0,alias:[t,"language-"+t],inside:r.languages[t]},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}}}})}}),r.languages.html=r.languages.markup,r.languages.mathml=r.languages.markup,r.languages.svg=r.languages.markup,r.languages.xml=r.languages.extend("markup",{}),r.languages.ssml=r.languages.xml,r.languages.atom=r.languages.xml,r.languages.rss=r.languages.xml,function(e){var t="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",n={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},s={bash:n,environment:{pattern:RegExp("\\$"+t),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--|\+\+|\*\*=?|<<=?|>>=?|&&|\|\||[=!+\-*/%<>^&|]=?|[?~:]/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+t),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{1,2})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)[\w-]+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b[\w-]+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+t),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:s},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:n}},{pattern:/(^|[^\\](?:\\\\)*)"(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|[^"\\`$])*"/,lookbehind:!0,greedy:!0,inside:s},{pattern:/(^|[^$\\])'[^']*'/,lookbehind:!0,greedy:!0},{pattern:/\$'(?:[^'\\]|\\[\s\S])*'/,greedy:!0,inside:{entity:s.entity}}],environment:{pattern:RegExp("\\$?"+t),alias:"constant"},variable:s.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|apt-cache|apt-get|aptitude|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|docker|docker-compose|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|node|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|podman|podman-compose|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vcpkg|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:case|do|done|elif|else|esac|fi|for|function|if|in|select|then|until|while)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|alias|bind|break|builtin|caller|cd|command|continue|declare|echo|enable|eval|exec|exit|export|getopts|hash|help|let|local|logout|mapfile|printf|pwd|read|readarray|readonly|return|set|shift|shopt|source|test|times|trap|type|typeset|ulimit|umask|unalias|unset)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:false|true)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|=[=~]?|!=?|<<[<-]?|[&\d]?>>|\d[<>]&?|[<>][&=]?|&[>&]?|\|[&|]?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},n.inside=e.languages.bash;for(var r=["comment","function-name","for-or-select","assign-left","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],o=s.variable[1].inside,a=0;a<r.length;a++)o[r[a]]=e.languages.bash[r[a]];e.languages.shell=e.languages.bash}(r),r.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|trait)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:break|catch|continue|do|else|finally|for|function|if|in|instanceof|new|null|return|throw|try|while)\b/,boolean:/\b(?:false|true)\b/,function:/\b\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/},r.languages.c=r.languages.extend("clike",{comment:{pattern:/\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},"class-name":{pattern:/(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/,lookbehind:!0},keyword:/\b(?:_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|__attribute__|asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|inline|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|typeof|union|unsigned|void|volatile|while)\b/,function:/\b[a-z_]\w*(?=\s*\()/i,number:/(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i,operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/}),r.languages.insertBefore("c","string",{char:{pattern:/'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n]){0,32}'/,greedy:!0}}),r.languages.insertBefore("c","string",{macro:{pattern:/(^[\t ]*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im,lookbehind:!0,greedy:!0,alias:"property",inside:{string:[{pattern:/^(#\s*include\s*)<[^>]+>/,lookbehind:!0},r.languages.c.string],char:r.languages.c.char,comment:r.languages.c.comment,"macro-name":[{pattern:/(^#\s*define\s+)\w+\b(?!\()/i,lookbehind:!0},{pattern:/(^#\s*define\s+)\w+\b(?=\()/i,lookbehind:!0,alias:"function"}],directive:{pattern:/^(#\s*)[a-z]+/,lookbehind:!0,alias:"keyword"},"directive-hash":/^#/,punctuation:/##|\\(?=[\r\n])/,expression:{pattern:/\S[\s\S]*/,inside:r.languages.c}}}}),r.languages.insertBefore("c","function",{constant:/\b(?:EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|__DATE__|__FILE__|__LINE__|__TIMESTAMP__|__TIME__|__func__|stderr|stdin|stdout)\b/}),delete r.languages.c.boolean,function(e){var t=/\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const|const_cast|consteval|constexpr|constinit|continue|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|final|float|for|friend|goto|if|import|inline|int|int16_t|int32_t|int64_t|int8_t|long|module|mutable|namespace|new|noexcept|nullptr|operator|override|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|uint16_t|uint32_t|uint64_t|uint8_t|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/,n=/\b(?!<keyword>)\w+(?:\s*\.\s*\w+)*\b/.source.replace(/<keyword>/g,(function(){return t.source}));e.languages.cpp=e.languages.extend("c",{"class-name":[{pattern:RegExp(/(\b(?:class|concept|enum|struct|typename)\s+)(?!<keyword>)\w+/.source.replace(/<keyword>/g,(function(){return t.source}))),lookbehind:!0},/\b[A-Z]\w*(?=\s*::\s*\w+\s*\()/,/\b[A-Z_]\w*(?=\s*::\s*~\w+\s*\()/i,/\b\w+(?=\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*::\s*\w+\s*\()/],keyword:t,number:{pattern:/(?:\b0b[01']+|\b0x(?:[\da-f']+(?:\.[\da-f']*)?|\.[\da-f']+)(?:p[+-]?[\d']+)?|(?:\b[\d']+(?:\.[\d']*)?|\B\.[\d']+)(?:e[+-]?[\d']+)?)[ful]{0,4}/i,greedy:!0},operator:/>>=?|<<=?|->|--|\+\+|&&|\|\||[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/,boolean:/\b(?:false|true)\b/}),e.languages.insertBefore("cpp","string",{module:{pattern:RegExp(/(\b(?:import|module)\s+)/.source+"(?:"+/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|<[^<>\r\n]*>/.source+"|"+/<mod-name>(?:\s*:\s*<mod-name>)?|:\s*<mod-name>/.source.replace(/<mod-name>/g,(function(){return n}))+")"),lookbehind:!0,greedy:!0,inside:{string:/^[<"][\s\S]+/,operator:/:/,punctuation:/\./}},"raw-string":{pattern:/R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/,alias:"string",greedy:!0}}),e.languages.insertBefore("cpp","keyword",{"generic-function":{pattern:/\b(?!operator\b)[a-z_]\w*\s*<(?:[^<>]|<[^<>]*>)*>(?=\s*\()/i,inside:{function:/^\w+/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:e.languages.cpp}}}}),e.languages.insertBefore("cpp","operator",{"double-colon":{pattern:/::/,alias:"punctuation"}}),e.languages.insertBefore("cpp","class-name",{"base-clause":{pattern:/(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/,lookbehind:!0,greedy:!0,inside:e.languages.extend("cpp",{})}}),e.languages.insertBefore("inside","double-colon",{"class-name":/\b[a-z_]\w*\b(?!\s*::)/i},e.languages.cpp["base-clause"])}(r),function(e){var t=/(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/;e.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-](?:[^;{\s]|\s+(?![\s{]))*(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+t.source+"|"+/(?:[^\\\r\n()"']|\\[\s\S])*/.source+")\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+t.source+"$"),alias:"url"}}},selector:{pattern:RegExp("(^|[{}\\s])[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+t.source+")*(?=\\s*\\{)"),lookbehind:!0},string:{pattern:t,greedy:!0},property:{pattern:/(^|[^-\w\xA0-\uFFFF])(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,lookbehind:!0},important:/!important\b/i,function:{pattern:/(^|[^-a-z0-9])[-a-z0-9]+(?=\()/i,lookbehind:!0},punctuation:/[(){};:,]/},e.languages.css.atrule.inside.rest=e.languages.css;var n=e.languages.markup;n&&(n.tag.addInlined("style","css"),n.tag.addAttribute("style","css"))}(r),function(e){var t,n=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css.selector={pattern:e.languages.css.selector.pattern,lookbehind:!0,inside:t={"pseudo-element":/:(?:after|before|first-letter|first-line|selection)|::[-\w]+/,"pseudo-class":/:[-\w]+/,class:/\.[-\w]+/,id:/#[-\w]+/,attribute:{pattern:RegExp("\\[(?:[^[\\]\"']|"+n.source+")*\\]"),greedy:!0,inside:{punctuation:/^\[|\]$/,"case-sensitivity":{pattern:/(\s)[si]$/i,lookbehind:!0,alias:"keyword"},namespace:{pattern:/^(\s*)(?:(?!\s)[-*\w\xA0-\uFFFF])*\|(?!=)/,lookbehind:!0,inside:{punctuation:/\|$/}},"attr-name":{pattern:/^(\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+/,lookbehind:!0},"attr-value":[n,{pattern:/(=\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+(?=\s*$)/,lookbehind:!0}],operator:/[|~*^$]?=/}},"n-th":[{pattern:/(\(\s*)[+-]?\d*[\dn](?:\s*[+-]\s*\d+)?(?=\s*\))/,lookbehind:!0,inside:{number:/[\dn]+/,operator:/[+-]/}},{pattern:/(\(\s*)(?:even|odd)(?=\s*\))/i,lookbehind:!0}],combinator:/>|\+|~|\|\|/,punctuation:/[(),]/}},e.languages.css.atrule.inside["selector-function-argument"].inside=t,e.languages.insertBefore("css","property",{variable:{pattern:/(^|[^-\w\xA0-\uFFFF])--(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*/i,lookbehind:!0}});var s={pattern:/(\b\d+)(?:%|[a-z]+(?![\w-]))/,lookbehind:!0},r={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0};e.languages.insertBefore("css","function",{operator:{pattern:/(\s)[+\-*\/](?=\s)/,lookbehind:!0},hexcode:{pattern:/\B#[\da-f]{3,8}\b/i,alias:"color"},color:[{pattern:/(^|[^\w-])(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)(?![\w-])/i,lookbehind:!0},{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:s,number:r,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:s,number:r})}(r),r.languages.javascript=r.languages.extend("clike",{"class-name":[r.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:constructor|prototype))/,lookbehind:!0}],keyword:[{pattern:/((?:^|\})\s*)catch\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|assert(?=\s*\{)|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally(?=\s*(?:\{|$))|for|from(?=\s*(?:['"]|$))|function|(?:get|set)(?=\s*(?:[#\[$\w\xA0-\uFFFF]|$))|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:{pattern:RegExp(/(^|[^\w$])/.source+"(?:"+/NaN|Infinity/.source+"|"+/0[bB][01]+(?:_[01]+)*n?/.source+"|"+/0[oO][0-7]+(?:_[0-7]+)*n?/.source+"|"+/0[xX][\dA-Fa-f]+(?:_[\dA-Fa-f]+)*n?/.source+"|"+/\d+(?:_\d+)*n/.source+"|"+/(?:\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\.\d+(?:_\d+)*)(?:[Ee][+-]?\d+(?:_\d+)*)?/.source+")"+/(?![\w$])/.source),lookbehind:!0},operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),r.languages.javascript["class-name"][0].pattern=/(\b(?:class|extends|implements|instanceof|interface|new)\s+)[\w.\\]+/,r.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:r.languages.regex},"regex-delimiter":/^\/|\/$/,"regex-flags":/^[a-z]+$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:r.languages.javascript},{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,lookbehind:!0,inside:r.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:r.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:r.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),r.languages.insertBefore("javascript","string",{hashbang:{pattern:/^#!.*/,greedy:!0,alias:"comment"},"template-string":{pattern:/`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\$\{|\}$/,alias:"punctuation"},rest:r.languages.javascript}},string:/[\s\S]+/}},"string-property":{pattern:/((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m,lookbehind:!0,greedy:!0,alias:"property"}}),r.languages.insertBefore("javascript","operator",{"literal-property":{pattern:/((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m,lookbehind:!0,alias:"property"}}),r.languages.markup&&(r.languages.markup.tag.addInlined("script","javascript"),r.languages.markup.tag.addAttribute(/on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)/.source,"javascript")),r.languages.js=r.languages.javascript,function(e){var t=/#(?!\{).+/,n={pattern:/#\{[^}]+\}/,alias:"variable"};e.languages.coffeescript=e.languages.extend("javascript",{comment:t,string:[{pattern:/'(?:\\[\s\S]|[^\\'])*'/,greedy:!0},{pattern:/"(?:\\[\s\S]|[^\\"])*"/,greedy:!0,inside:{interpolation:n}}],keyword:/\b(?:and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield)\b/,"class-member":{pattern:/@(?!\d)\w+/,alias:"variable"}}),e.languages.insertBefore("coffeescript","comment",{"multiline-comment":{pattern:/###[\s\S]+?###/,alias:"comment"},"block-regex":{pattern:/\/{3}[\s\S]*?\/{3}/,alias:"regex",inside:{comment:t,interpolation:n}}}),e.languages.insertBefore("coffeescript","string",{"inline-javascript":{pattern:/`(?:\\[\s\S]|[^\\`])*`/,inside:{delimiter:{pattern:/^`|`$/,alias:"punctuation"},script:{pattern:/[\s\S]+/,alias:"language-javascript",inside:e.languages.javascript}}},"multiline-string":[{pattern:/'''[\s\S]*?'''/,greedy:!0,alias:"string"},{pattern:/"""[\s\S]*?"""/,greedy:!0,alias:"string",inside:{interpolation:n}}]}),e.languages.insertBefore("coffeescript","keyword",{property:/(?!\d)\w+(?=\s*:(?!:))/}),delete e.languages.coffeescript["template-string"],e.languages.coffee=e.languages.coffeescript}(r),function(e){var t=/[*&][^\s[\]{},]+/,n=/!(?:<[\w\-%#;/?:@&=+$,.!~*'()[\]]+>|(?:[a-zA-Z\d-]*!)?[\w\-%#;/?:@&=+$.~*'()]+)?/,s="(?:"+n.source+"(?:[ \t]+"+t.source+")?|"+t.source+"(?:[ \t]+"+n.source+")?)",r=/(?:[^\s\x00-\x08\x0e-\x1f!"#%&'*,\-:>?@[\]`{|}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]|[?:-]<PLAIN>)(?:[ \t]*(?:(?![#:])<PLAIN>|:<PLAIN>))*/.source.replace(/<PLAIN>/g,(function(){return/[^\s\x00-\x08\x0e-\x1f,[\]{}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]/.source})),o=/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/.source;function a(e,t){t=(t||"").replace(/m/g,"")+"m";var n=/([:\-,[{]\s*(?:\s<<prop>>[ \t]+)?)(?:<<value>>)(?=[ \t]*(?:$|,|\]|\}|(?:[\r\n]\s*)?#))/.source.replace(/<<prop>>/g,(function(){return s})).replace(/<<value>>/g,(function(){return e}));return RegExp(n,t)}e.languages.yaml={scalar:{pattern:RegExp(/([\-:]\s*(?:\s<<prop>>[ \t]+)?[|>])[ \t]*(?:((?:\r?\n|\r)[ \t]+)\S[^\r\n]*(?:\2[^\r\n]+)*)/.source.replace(/<<prop>>/g,(function(){return s}))),lookbehind:!0,alias:"string"},comment:/#.*/,key:{pattern:RegExp(/((?:^|[:\-,[{\r\n?])[ \t]*(?:<<prop>>[ \t]+)?)<<key>>(?=\s*:\s)/.source.replace(/<<prop>>/g,(function(){return s})).replace(/<<key>>/g,(function(){return"(?:"+r+"|"+o+")"}))),lookbehind:!0,greedy:!0,alias:"atrule"},directive:{pattern:/(^[ \t]*)%.+/m,lookbehind:!0,alias:"important"},datetime:{pattern:a(/\d{4}-\d\d?-\d\d?(?:[tT]|[ \t]+)\d\d?:\d{2}:\d{2}(?:\.\d*)?(?:[ \t]*(?:Z|[-+]\d\d?(?::\d{2})?))?|\d{4}-\d{2}-\d{2}|\d\d?:\d{2}(?::\d{2}(?:\.\d*)?)?/.source),lookbehind:!0,alias:"number"},boolean:{pattern:a(/false|true/.source,"i"),lookbehind:!0,alias:"important"},null:{pattern:a(/null|~/.source,"i"),lookbehind:!0,alias:"important"},string:{pattern:a(o),lookbehind:!0,greedy:!0},number:{pattern:a(/[+-]?(?:0x[\da-f]+|0o[0-7]+|(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|\.inf|\.nan)/.source,"i"),lookbehind:!0},tag:n,important:t,punctuation:/---|[:[\]{}\-,|>?]|\.\.\./},e.languages.yml=e.languages.yaml}(r),function(e){var t=/(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?![\r\n]))/.source;function n(e){return e=e.replace(/<inner>/g,(function(){return t})),RegExp(/((?:^|[^\\])(?:\\{2})*)/.source+"(?:"+e+")")}var s=/(?:\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\|\r\n`])+/.source,r=/\|?__(?:\|__)+\|?(?:(?:\n|\r\n?)|(?![\s\S]))/.source.replace(/__/g,(function(){return s})),o=/\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/.source;e.languages.markdown=e.languages.extend("markup",{}),e.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"front-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:e.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+r+o+"(?:"+r+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+r+o+")(?:"+r+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(s),inside:e.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+r+")"+o+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+r+"$"),inside:{"table-header":{pattern:RegExp(s),alias:"important",inside:e.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n(/\b__(?:(?!_)<inner>|_(?:(?!_)<inner>)+_)+__\b|\*\*(?:(?!\*)<inner>|\*(?:(?!\*)<inner>)+\*)+\*\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n(/\b_(?:(?!_)<inner>|__(?:(?!_)<inner>)+__)+_\b|\*(?:(?!\*)<inner>|\*\*(?:(?!\*)<inner>)+\*\*)+\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n(/(~~?)(?:(?!~)<inner>)+\2/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},"code-snippet":{pattern:/(^|[^\\`])(?:``[^`\r\n]+(?:`[^`\r\n]+)*``(?!`)|`[^`\r\n]+`(?!`))/,lookbehind:!0,greedy:!0,alias:["code","keyword"]},url:{pattern:n(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(t){["url","bold","italic","strike","code-snippet"].forEach((function(n){t!==n&&(e.languages.markdown[t].inside.content.inside[n]=e.languages.markdown[n])}))})),e.hooks.add("after-tokenize",(function(e){"markdown"!==e.language&&"md"!==e.language||function e(t){if(t&&"string"!=typeof t)for(var n=0,s=t.length;n<s;n++){var r=t[n];if("code"===r.type){var o=r.content[1],a=r.content[3];if(o&&a&&"code-language"===o.type&&"code-block"===a.type&&"string"==typeof o.content){var i=o.content.replace(/\b#/g,"sharp").replace(/\b\+\+/g,"pp"),c="language-"+(i=(/[a-z][\w-]*/i.exec(i)||[""])[0].toLowerCase());a.alias?"string"==typeof a.alias?a.alias=[a.alias,c]:a.alias.push(c):a.alias=[c]}}else e(r.content)}}(e.tokens)})),e.hooks.add("wrap",(function(t){if("code-block"===t.type){for(var n="",s=0,r=t.classes.length;s<r;s++){var o=t.classes[s],l=/language-(.+)/.exec(o);if(l){n=l[1];break}}var d,p=e.languages[n];if(p)t.content=e.highlight((d=t.content,d.replace(a,"").replace(/&(\w{1,8}|#x?[\da-f]{1,8});/gi,(function(e,t){var n;if("#"===(t=t.toLowerCase())[0])return n="x"===t[1]?parseInt(t.slice(2),16):Number(t.slice(1)),c(n);var s=i[t];return s||e}))),p,n);else if(n&&"none"!==n&&e.plugins.autoloader){var u="md-"+(new Date).valueOf()+"-"+Math.floor(1e16*Math.random());t.attributes.id=u,e.plugins.autoloader.loadLanguages(n,(function(){var t=document.getElementById(u);t&&(t.innerHTML=e.highlight(t.textContent,e.languages[n],n))}))}}}));var a=RegExp(e.languages.markup.tag.pattern.source,"gi"),i={amp:"&",lt:"<",gt:">",quot:'"'},c=String.fromCodePoint||String.fromCharCode;e.languages.md=e.languages.markdown}(r),r.languages.graphql={comment:/#.*/,description:{pattern:/(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i,greedy:!0,alias:"string",inside:{"language-markdown":{pattern:/(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/,lookbehind:!0,inside:r.languages.markdown}}},string:{pattern:/"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/,greedy:!0},number:/(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,boolean:/\b(?:false|true)\b/,variable:/\$[a-z_]\w*/i,directive:{pattern:/@[a-z_]\w*/i,alias:"function"},"attr-name":{pattern:/\b[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i,greedy:!0},"atom-input":{pattern:/\b[A-Z]\w*Input\b/,alias:"class-name"},scalar:/\b(?:Boolean|Float|ID|Int|String)\b/,constant:/\b[A-Z][A-Z_\d]*\b/,"class-name":{pattern:/(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*|:\s*|\[)[A-Z_]\w*/,lookbehind:!0},fragment:{pattern:/(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-mutation":{pattern:/(\bmutation\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-query":{pattern:/(\bquery\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},keyword:/\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/,operator:/[!=|&]|\.{3}/,"property-query":/\w+(?=\s*\()/,object:/\w+(?=\s*\{)/,punctuation:/[!(){}\[\]:=,]/,property:/\w+/},r.hooks.add("after-tokenize",(function(e){if("graphql"===e.language)for(var t=e.tokens.filter((function(e){return"string"!=typeof e&&"comment"!==e.type&&"scalar"!==e.type})),n=0;n<t.length;){var s=t[n++];if("keyword"===s.type&&"mutation"===s.content){var r=[];if(p(["definition-mutation","punctuation"])&&"("===d(1).content){n+=2;var o=u(/^\($/,/^\)$/);if(-1===o)continue;for(;n<o;n++){var a=d(0);"variable"===a.type&&(m(a,"variable-input"),r.push(a.content))}n=o+1}if(p(["punctuation","property-query"])&&"{"===d(0).content&&(n++,m(d(0),"property-mutation"),r.length>0)){var i=u(/^\{$/,/^\}$/);if(-1===i)continue;for(var c=n;c<i;c++){var l=t[c];"variable"===l.type&&r.indexOf(l.content)>=0&&m(l,"variable-input")}}}}function d(e){return t[n+e]}function p(e,t){t=t||0;for(var n=0;n<e.length;n++){var s=d(n+t);if(!s||s.type!==e[n])return!1}return!0}function u(e,s){for(var r=1,o=n;o<t.length;o++){var a=t[o],i=a.content;if("punctuation"===a.type&&"string"==typeof i)if(e.test(i))r++;else if(s.test(i)&&0===--r)return o}return-1}function m(e,t){var n=e.alias;n?Array.isArray(n)||(e.alias=n=[n]):e.alias=n=[],n.push(t)}})),r.languages.sql={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,lookbehind:!0},variable:[{pattern:/@(["'`])(?:\\[\s\S]|(?!\1)[^\\])+\1/,greedy:!0},/@[\w.$]+/],string:{pattern:/(^|[^@\\])("|')(?:\\[\s\S]|(?!\2)[^\\]|\2\2)*\2/,greedy:!0,lookbehind:!0},identifier:{pattern:/(^|[^@\\])`(?:\\[\s\S]|[^`\\]|``)*`/,greedy:!0,lookbehind:!0,inside:{punctuation:/^`|`$/}},function:/\b(?:AVG|COUNT|FIRST|FORMAT|LAST|LCASE|LEN|MAX|MID|MIN|MOD|NOW|ROUND|SUM|UCASE)(?=\s*\()/i,keyword:/\b(?:ACTION|ADD|AFTER|ALGORITHM|ALL|ALTER|ANALYZE|ANY|APPLY|AS|ASC|AUTHORIZATION|AUTO_INCREMENT|BACKUP|BDB|BEGIN|BERKELEYDB|BIGINT|BINARY|BIT|BLOB|BOOL|BOOLEAN|BREAK|BROWSE|BTREE|BULK|BY|CALL|CASCADED?|CASE|CHAIN|CHAR(?:ACTER|SET)?|CHECK(?:POINT)?|CLOSE|CLUSTERED|COALESCE|COLLATE|COLUMNS?|COMMENT|COMMIT(?:TED)?|COMPUTE|CONNECT|CONSISTENT|CONSTRAINT|CONTAINS(?:TABLE)?|CONTINUE|CONVERT|CREATE|CROSS|CURRENT(?:_DATE|_TIME|_TIMESTAMP|_USER)?|CURSOR|CYCLE|DATA(?:BASES?)?|DATE(?:TIME)?|DAY|DBCC|DEALLOCATE|DEC|DECIMAL|DECLARE|DEFAULT|DEFINER|DELAYED|DELETE|DELIMITERS?|DENY|DESC|DESCRIBE|DETERMINISTIC|DISABLE|DISCARD|DISK|DISTINCT|DISTINCTROW|DISTRIBUTED|DO|DOUBLE|DROP|DUMMY|DUMP(?:FILE)?|DUPLICATE|ELSE(?:IF)?|ENABLE|ENCLOSED|END|ENGINE|ENUM|ERRLVL|ERRORS|ESCAPED?|EXCEPT|EXEC(?:UTE)?|EXISTS|EXIT|EXPLAIN|EXTENDED|FETCH|FIELDS|FILE|FILLFACTOR|FIRST|FIXED|FLOAT|FOLLOWING|FOR(?: EACH ROW)?|FORCE|FOREIGN|FREETEXT(?:TABLE)?|FROM|FULL|FUNCTION|GEOMETRY(?:COLLECTION)?|GLOBAL|GOTO|GRANT|GROUP|HANDLER|HASH|HAVING|HOLDLOCK|HOUR|IDENTITY(?:COL|_INSERT)?|IF|IGNORE|IMPORT|INDEX|INFILE|INNER|INNODB|INOUT|INSERT|INT|INTEGER|INTERSECT|INTERVAL|INTO|INVOKER|ISOLATION|ITERATE|JOIN|KEYS?|KILL|LANGUAGE|LAST|LEAVE|LEFT|LEVEL|LIMIT|LINENO|LINES|LINESTRING|LOAD|LOCAL|LOCK|LONG(?:BLOB|TEXT)|LOOP|MATCH(?:ED)?|MEDIUM(?:BLOB|INT|TEXT)|MERGE|MIDDLEINT|MINUTE|MODE|MODIFIES|MODIFY|MONTH|MULTI(?:LINESTRING|POINT|POLYGON)|NATIONAL|NATURAL|NCHAR|NEXT|NO|NONCLUSTERED|NULLIF|NUMERIC|OFF?|OFFSETS?|ON|OPEN(?:DATASOURCE|QUERY|ROWSET)?|OPTIMIZE|OPTION(?:ALLY)?|ORDER|OUT(?:ER|FILE)?|OVER|PARTIAL|PARTITION|PERCENT|PIVOT|PLAN|POINT|POLYGON|PRECEDING|PRECISION|PREPARE|PREV|PRIMARY|PRINT|PRIVILEGES|PROC(?:EDURE)?|PUBLIC|PURGE|QUICK|RAISERROR|READS?|REAL|RECONFIGURE|REFERENCES|RELEASE|RENAME|REPEAT(?:ABLE)?|REPLACE|REPLICATION|REQUIRE|RESIGNAL|RESTORE|RESTRICT|RETURN(?:ING|S)?|REVOKE|RIGHT|ROLLBACK|ROUTINE|ROW(?:COUNT|GUIDCOL|S)?|RTREE|RULE|SAVE(?:POINT)?|SCHEMA|SECOND|SELECT|SERIAL(?:IZABLE)?|SESSION(?:_USER)?|SET(?:USER)?|SHARE|SHOW|SHUTDOWN|SIMPLE|SMALLINT|SNAPSHOT|SOME|SONAME|SQL|START(?:ING)?|STATISTICS|STATUS|STRIPED|SYSTEM_USER|TABLES?|TABLESPACE|TEMP(?:ORARY|TABLE)?|TERMINATED|TEXT(?:SIZE)?|THEN|TIME(?:STAMP)?|TINY(?:BLOB|INT|TEXT)|TOP?|TRAN(?:SACTIONS?)?|TRIGGER|TRUNCATE|TSEQUAL|TYPES?|UNBOUNDED|UNCOMMITTED|UNDEFINED|UNION|UNIQUE|UNLOCK|UNPIVOT|UNSIGNED|UPDATE(?:TEXT)?|USAGE|USE|USER|USING|VALUES?|VAR(?:BINARY|CHAR|CHARACTER|YING)|VIEW|WAITFOR|WARNINGS|WHEN|WHERE|WHILE|WITH(?: ROLLUP|IN)?|WORK|WRITE(?:TEXT)?|YEAR)\b/i,boolean:/\b(?:FALSE|NULL|TRUE)\b/i,number:/\b0x[\da-f]+\b|\b\d+(?:\.\d*)?|\B\.\d+\b/i,operator:/[-+*\/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?|\b(?:AND|BETWEEN|DIV|ILIKE|IN|IS|LIKE|NOT|OR|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b/i,punctuation:/[;[\]()`,.]/},function(e){var t=e.languages.javascript["template-string"],n=t.pattern.source,s=t.inside.interpolation,r=s.inside["interpolation-punctuation"],o=s.pattern.source;function a(t,s){if(e.languages[t])return{pattern:RegExp("((?:"+s+")\\s*)"+n),lookbehind:!0,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},"embedded-code":{pattern:/[\s\S]+/,alias:t}}}}function i(e,t){return"___"+t.toUpperCase()+"_"+e+"___"}function c(t,n,s){var r={code:t,grammar:n,language:s};return e.hooks.run("before-tokenize",r),r.tokens=e.tokenize(r.code,r.grammar),e.hooks.run("after-tokenize",r),r.tokens}function l(t){var n={};n["interpolation-punctuation"]=r;var o=e.tokenize(t,n);if(3===o.length){var a=[1,1];a.push.apply(a,c(o[1],e.languages.javascript,"javascript")),o.splice.apply(o,a)}return new e.Token("interpolation",o,s.alias,t)}function d(t,n,s){var r=e.tokenize(t,{interpolation:{pattern:RegExp(o),lookbehind:!0}}),a=0,d={},p=c(r.map((function(e){if("string"==typeof e)return e;for(var n,r=e.content;-1!==t.indexOf(n=i(a++,s)););return d[n]=r,n})).join(""),n,s),u=Object.keys(d);return a=0,function e(t){for(var n=0;n<t.length;n++){if(a>=u.length)return;var s=t[n];if("string"==typeof s||"string"==typeof s.content){var r=u[a],o="string"==typeof s?s:s.content,i=o.indexOf(r);if(-1!==i){++a;var c=o.substring(0,i),p=l(d[r]),m=o.substring(i+r.length),h=[];if(c&&h.push(c),h.push(p),m){var g=[m];e(g),h.push.apply(h,g)}"string"==typeof s?(t.splice.apply(t,[n,1].concat(h)),n+=h.length-1):s.content=h}}else{var f=s.content;Array.isArray(f)?e(f):e([f])}}}(p),new e.Token(s,p,"language-"+s,t)}e.languages.javascript["template-string"]=[a("css",/\b(?:styled(?:\([^)]*\))?(?:\s*\.\s*\w+(?:\([^)]*\))*)*|css(?:\s*\.\s*(?:global|resolve))?|createGlobalStyle|keyframes)/.source),a("html",/\bhtml|\.\s*(?:inner|outer)HTML\s*\+?=/.source),a("svg",/\bsvg/.source),a("markdown",/\b(?:markdown|md)/.source),a("graphql",/\b(?:gql|graphql(?:\s*\.\s*experimental)?)/.source),a("sql",/\bsql/.source),t].filter(Boolean);var p={javascript:!0,js:!0,typescript:!0,ts:!0,jsx:!0,tsx:!0};function u(e){return"string"==typeof e?e:Array.isArray(e)?e.map(u).join(""):u(e.content)}e.hooks.add("after-tokenize",(function(t){t.language in p&&function t(n){for(var s=0,r=n.length;s<r;s++){var o=n[s];if("string"!=typeof o){var a=o.content;if(Array.isArray(a))if("template-string"===o.type){var i=a[1];if(3===a.length&&"string"!=typeof i&&"embedded-code"===i.type){var c=u(i),l=i.alias,p=Array.isArray(l)?l[0]:l,m=e.languages[p];if(!m)continue;a[1]=d(c,m,p)}}else t(a);else"string"!=typeof a&&t([a])}}}(t.tokens)}))}(r),function(e){e.languages.typescript=e.languages.extend("javascript",{"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|type)\s+)(?!keyof\b)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?:\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},builtin:/\b(?:Array|Function|Promise|any|boolean|console|never|number|string|symbol|unknown)\b/}),e.languages.typescript.keyword.push(/\b(?:abstract|declare|is|keyof|readonly|require)\b/,/\b(?:asserts|infer|interface|module|namespace|type)\b(?=\s*(?:[{_$a-zA-Z\xA0-\uFFFF]|$))/,/\btype\b(?=\s*(?:[\{*]|$))/),delete e.languages.typescript.parameter,delete e.languages.typescript["literal-property"];var t=e.languages.extend("typescript",{});delete t["class-name"],e.languages.typescript["class-name"].inside=t,e.languages.insertBefore("typescript","function",{decorator:{pattern:/@[$\w\xA0-\uFFFF]+/,inside:{at:{pattern:/^@/,alias:"operator"},function:/^[\s\S]+/}},"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:t}}}}),e.languages.ts=e.languages.typescript}(r),function(e){function t(e,t){return RegExp(e.replace(/<ID>/g,(function(){return/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/.source})),t)}e.languages.insertBefore("javascript","function-variable",{"method-variable":{pattern:RegExp("(\\.\\s*)"+e.languages.javascript["function-variable"].pattern.source),lookbehind:!0,alias:["function-variable","method","function","property-access"]}}),e.languages.insertBefore("javascript","function",{method:{pattern:RegExp("(\\.\\s*)"+e.languages.javascript.function.source),lookbehind:!0,alias:["function","property-access"]}}),e.languages.insertBefore("javascript","constant",{"known-class-name":[{pattern:/\b(?:(?:Float(?:32|64)|(?:Int|Uint)(?:8|16|32)|Uint8Clamped)?Array|ArrayBuffer|BigInt|Boolean|DataView|Date|Error|Function|Intl|JSON|(?:Weak)?(?:Map|Set)|Math|Number|Object|Promise|Proxy|Reflect|RegExp|String|Symbol|WebAssembly)\b/,alias:"class-name"},{pattern:/\b(?:[A-Z]\w*)Error\b/,alias:"class-name"}]}),e.languages.insertBefore("javascript","keyword",{imports:{pattern:t(/(\bimport\b\s*)(?:<ID>(?:\s*,\s*(?:\*\s*as\s+<ID>|\{[^{}]*\}))?|\*\s*as\s+<ID>|\{[^{}]*\})(?=\s*\bfrom\b)/.source),lookbehind:!0,inside:e.languages.javascript},exports:{pattern:t(/(\bexport\b\s*)(?:\*(?:\s*as\s+<ID>)?(?=\s*\bfrom\b)|\{[^{}]*\})/.source),lookbehind:!0,inside:e.languages.javascript}}),e.languages.javascript.keyword.unshift({pattern:/\b(?:as|default|export|from|import)\b/,alias:"module"},{pattern:/\b(?:await|break|catch|continue|do|else|finally|for|if|return|switch|throw|try|while|yield)\b/,alias:"control-flow"},{pattern:/\bnull\b/,alias:["null","nil"]},{pattern:/\bundefined\b/,alias:"nil"}),e.languages.insertBefore("javascript","operator",{spread:{pattern:/\.{3}/,alias:"operator"},arrow:{pattern:/=>/,alias:"operator"}}),e.languages.insertBefore("javascript","punctuation",{"property-access":{pattern:t(/(\.\s*)#?<ID>/.source),lookbehind:!0},"maybe-class-name":{pattern:/(^|[^$\w\xA0-\uFFFF])[A-Z][$\w\xA0-\uFFFF]+/,lookbehind:!0},dom:{pattern:/\b(?:document|(?:local|session)Storage|location|navigator|performance|window)\b/,alias:"variable"},console:{pattern:/\bconsole(?=\s*\.)/,alias:"class-name"}});for(var n=["function","function-variable","method","method-variable","property-access"],s=0;s<n.length;s++){var r=n[s],o=e.languages.javascript[r];"RegExp"===e.util.type(o)&&(o=e.languages.javascript[r]={pattern:o});var a=o.inside||{};o.inside=a,a["maybe-class-name"]=/^[A-Z][\s\S]*/}}(r),function(e){var t=e.util.clone(e.languages.javascript),n=/(?:\s|\/\/.*(?!.)|\/\*(?:[^*]|\*(?!\/))\*\/)/.source,s=/(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])*\})/.source,r=/(?:\{<S>*\.{3}(?:[^{}]|<BRACES>)*\})/.source;function o(e,t){return e=e.replace(/<S>/g,(function(){return n})).replace(/<BRACES>/g,(function(){return s})).replace(/<SPREAD>/g,(function(){return r})),RegExp(e,t)}r=o(r).source,e.languages.jsx=e.languages.extend("markup",t),e.languages.jsx.tag.pattern=o(/<\/?(?:[\w.:-]+(?:<S>+(?:[\w.:$-]+(?:=(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s{'"/>=]+|<BRACES>))?|<SPREAD>))*<S>*\/?)?>/.source),e.languages.jsx.tag.inside.tag.pattern=/^<\/?[^\s>\/]*/,e.languages.jsx.tag.inside["attr-value"].pattern=/=(?!\{)(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s'">]+)/,e.languages.jsx.tag.inside.tag.inside["class-name"]=/^[A-Z]\w*(?:\.[A-Z]\w*)*$/,e.languages.jsx.tag.inside.comment=t.comment,e.languages.insertBefore("inside","attr-name",{spread:{pattern:o(/<SPREAD>/.source),inside:e.languages.jsx}},e.languages.jsx.tag),e.languages.insertBefore("inside","special-attr",{script:{pattern:o(/=<BRACES>/.source),alias:"language-javascript",inside:{"script-punctuation":{pattern:/^=(?=\{)/,alias:"punctuation"},rest:e.languages.jsx}}},e.languages.jsx.tag);var a=function(e){return e?"string"==typeof e?e:"string"==typeof e.content?e.content:e.content.map(a).join(""):""},i=function(t){for(var n=[],s=0;s<t.length;s++){var r=t[s],o=!1;if("string"!=typeof r&&("tag"===r.type&&r.content[0]&&"tag"===r.content[0].type?"</"===r.content[0].content[0].content?n.length>0&&n[n.length-1].tagName===a(r.content[0].content[1])&&n.pop():"/>"===r.content[r.content.length-1].content||n.push({tagName:a(r.content[0].content[1]),openedBraces:0}):n.length>0&&"punctuation"===r.type&&"{"===r.content?n[n.length-1].openedBraces++:n.length>0&&n[n.length-1].openedBraces>0&&"punctuation"===r.type&&"}"===r.content?n[n.length-1].openedBraces--:o=!0),(o||"string"==typeof r)&&n.length>0&&0===n[n.length-1].openedBraces){var c=a(r);s<t.length-1&&("string"==typeof t[s+1]||"plain-text"===t[s+1].type)&&(c+=a(t[s+1]),t.splice(s+1,1)),s>0&&("string"==typeof t[s-1]||"plain-text"===t[s-1].type)&&(c=a(t[s-1])+c,t.splice(s-1,1),s--),t[s]=new e.Token("plain-text",c,null,c)}r.content&&"string"!=typeof r.content&&i(r.content)}};e.hooks.add("after-tokenize",(function(e){"jsx"!==e.language&&"tsx"!==e.language||i(e.tokens)}))}(r),function(e){e.languages.diff={coord:[/^(?:\*{3}|-{3}|\+{3}).*$/m,/^@@.*@@$/m,/^\d.*$/m]};var t={"deleted-sign":"-","deleted-arrow":"<","inserted-sign":"+","inserted-arrow":">",unchanged:" ",diff:"!"};Object.keys(t).forEach((function(n){var s=t[n],r=[];/^\w+$/.test(n)||r.push(/\w+/.exec(n)[0]),"diff"===n&&r.push("bold"),e.languages.diff[n]={pattern:RegExp("^(?:["+s+"].*(?:\r\n?|\n|(?![\\s\\S])))+","m"),alias:r,inside:{line:{pattern:/(.)(?=[\s\S]).*(?:\r\n?|\n)?/,lookbehind:!0},prefix:{pattern:/[\s\S]/,alias:/\w+/.exec(n)[0]}}}})),Object.defineProperty(e.languages.diff,"PREFIXES",{value:t})}(r),r.languages.git={comment:/^#.*/m,deleted:/^[-\u2013].*/m,inserted:/^\+.*/m,string:/("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,command:{pattern:/^.*\$ git .*$/m,inside:{parameter:/\s--?\w+/}},coord:/^@@.*@@$/m,"commit-sha1":/^commit \w{40}$/m},r.languages.go=r.languages.extend("clike",{string:{pattern:/(^|[^\\])"(?:\\.|[^"\\\r\n])*"|`[^`]*`/,lookbehind:!0,greedy:!0},keyword:/\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,boolean:/\b(?:_|false|iota|nil|true)\b/,number:[/\b0(?:b[01_]+|o[0-7_]+)i?\b/i,/\b0x(?:[a-f\d_]+(?:\.[a-f\d_]*)?|\.[a-f\d_]+)(?:p[+-]?\d+(?:_\d+)*)?i?(?!\w)/i,/(?:\b\d[\d_]*(?:\.[\d_]*)?|\B\.\d[\d_]*)(?:e[+-]?[\d_]+)?i?(?!\w)/i],operator:/[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,builtin:/\b(?:append|bool|byte|cap|close|complex|complex(?:64|128)|copy|delete|error|float(?:32|64)|u?int(?:8|16|32|64)?|imag|len|make|new|panic|print(?:ln)?|real|recover|rune|string|uintptr)\b/}),r.languages.insertBefore("go","string",{char:{pattern:/'(?:\\.|[^'\\\r\n]){0,10}'/,greedy:!0}}),delete r.languages.go["class-name"],function(e){function t(e,t){return"___"+e.toUpperCase()+t+"___"}Object.defineProperties(e.languages["markup-templating"]={},{buildPlaceholders:{value:function(n,s,r,o){if(n.language===s){var a=n.tokenStack=[];n.code=n.code.replace(r,(function(e){if("function"==typeof o&&!o(e))return e;for(var r,i=a.length;-1!==n.code.indexOf(r=t(s,i));)++i;return a[i]=e,r})),n.grammar=e.languages.markup}}},tokenizePlaceholders:{value:function(n,s){if(n.language===s&&n.tokenStack){n.grammar=e.languages[s];var r=0,o=Object.keys(n.tokenStack);!function a(i){for(var c=0;c<i.length&&!(r>=o.length);c++){var l=i[c];if("string"==typeof l||l.content&&"string"==typeof l.content){var d=o[r],p=n.tokenStack[d],u="string"==typeof l?l:l.content,m=t(s,d),h=u.indexOf(m);if(h>-1){++r;var g=u.substring(0,h),f=new e.Token(s,e.tokenize(p,n.grammar),"language-"+s,p),b=u.substring(h+m.length),y=[];g&&y.push.apply(y,a([g])),y.push(f),b&&y.push.apply(y,a([b])),"string"==typeof l?i.splice.apply(i,[c,1].concat(y)):l.content=y}}else l.content&&a(l.content)}return i}(n.tokens)}}}})}(r),function(e){e.languages.handlebars={comment:/\{\{![\s\S]*?\}\}/,delimiter:{pattern:/^\{\{\{?|\}\}\}?$/,alias:"punctuation"},string:/(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][+-]?\d+)?/,boolean:/\b(?:false|true)\b/,block:{pattern:/^(\s*(?:~\s*)?)[#\/]\S+?(?=\s*(?:~\s*)?$|\s)/,lookbehind:!0,alias:"keyword"},brackets:{pattern:/\[[^\]]+\]/,inside:{punctuation:/\[|\]/,variable:/[\s\S]+/}},punctuation:/[!"#%&':()*+,.\/;<=>@\[\\\]^`{|}~]/,variable:/[^!"#%&'()*+,\/;<=>@\[\\\]^`{|}~\s]+/},e.hooks.add("before-tokenize",(function(t){e.languages["markup-templating"].buildPlaceholders(t,"handlebars",/\{\{\{[\s\S]+?\}\}\}|\{\{[\s\S]+?\}\}/g)})),e.hooks.add("after-tokenize",(function(t){e.languages["markup-templating"].tokenizePlaceholders(t,"handlebars")})),e.languages.hbs=e.languages.handlebars}(r),r.languages.json={property:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?=\s*:)/,lookbehind:!0,greedy:!0},string:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?!\s*:)/,lookbehind:!0,greedy:!0},comment:{pattern:/\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},number:/-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,punctuation:/[{}[\],]/,operator:/:/,boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"}},r.languages.webmanifest=r.languages.json,r.languages.less=r.languages.extend("css",{comment:[/\/\*[\s\S]*?\*\//,{pattern:/(^|[^\\])\/\/.*/,lookbehind:!0}],atrule:{pattern:/@[\w-](?:\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{punctuation:/[:()]/}},selector:{pattern:/(?:@\{[\w-]+\}|[^{};\s@])(?:@\{[\w-]+\}|\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};@\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{variable:/@+[\w-]+/}},property:/(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/,operator:/[+\-*\/]/}),r.languages.insertBefore("less","property",{variable:[{pattern:/@[\w-]+\s*:/,inside:{punctuation:/:/}},/@@?[\w-]+/],"mixin-usage":{pattern:/([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/,lookbehind:!0,alias:"function"}}),r.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|[^\\\r\n])*/,lookbehind:!0},string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"builtin-target":{pattern:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,alias:"builtin"},target:{pattern:/^(?:[^:=\s]|[ \t]+(?![\s:]))+(?=\s*:(?!=))/m,alias:"symbol",inside:{variable:/\$+(?:(?!\$)[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:(?!\$)[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,function:{pattern:/(\()(?:abspath|addsuffix|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:list|s)?)(?=[ \t])/,lookbehind:!0},operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/},r.languages.objectivec=r.languages.extend("c",{string:{pattern:/@?"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},keyword:/\b(?:asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|in|inline|int|long|register|return|self|short|signed|sizeof|static|struct|super|switch|typedef|typeof|union|unsigned|void|volatile|while)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/,operator:/-[->]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|\|?|[~^%?*\/@]/}),delete r.languages.objectivec["class-name"],r.languages.objc=r.languages.objectivec,r.languages.ocaml={comment:{pattern:/\(\*[\s\S]*?\*\)/,greedy:!0},char:{pattern:/'(?:[^\\\r\n']|\\(?:.|[ox]?[0-9a-f]{1,3}))'/i,greedy:!0},string:[{pattern:/"(?:\\(?:[\s\S]|\r\n)|[^\\\r\n"])*"/,greedy:!0},{pattern:/\{([a-z_]*)\|[\s\S]*?\|\1\}/,greedy:!0}],number:[/\b(?:0b[01][01_]*|0o[0-7][0-7_]*)\b/i,/\b0x[a-f0-9][a-f0-9_]*(?:\.[a-f0-9_]*)?(?:p[+-]?\d[\d_]*)?(?!\w)/i,/\b\d[\d_]*(?:\.[\d_]*)?(?:e[+-]?\d[\d_]*)?(?!\w)/i],directive:{pattern:/\B#\w+/,alias:"property"},label:{pattern:/\B~\w+/,alias:"property"},"type-variable":{pattern:/\B'\w+/,alias:"function"},variant:{pattern:/`\w+/,alias:"symbol"},keyword:/\b(?:as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|module|mutable|new|nonrec|object|of|open|private|rec|sig|struct|then|to|try|type|val|value|virtual|when|where|while|with)\b/,boolean:/\b(?:false|true)\b/,"operator-like-punctuation":{pattern:/\[[<>|]|[>|]\]|\{<|>\}/,alias:"punctuation"},operator:/\.[.~]|:[=>]|[=<>@^|&+\-*\/$%!?~][!$%&*+\-.\/:<=>?@^|~]*|\b(?:and|asr|land|lor|lsl|lsr|lxor|mod|or)\b/,punctuation:/;;|::|[(){}\[\].,:;#]|\b_\b/},r.languages.python={comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0,greedy:!0},"string-interpolation":{pattern:/(?:f|fr|rf)(?:("""|''')[\s\S]*?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,greedy:!0,inside:{interpolation:{pattern:/((?:^|[^{])(?:\{\{)*)\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}])+\})+\})+\}/,lookbehind:!0,inside:{"format-spec":{pattern:/(:)[^:(){}]+(?=\}$)/,lookbehind:!0},"conversion-option":{pattern:/![sra](?=[:}]$)/,alias:"punctuation"},rest:null}},string:/[\s\S]+/}},"triple-quoted-string":{pattern:/(?:[rub]|br|rb)?("""|''')[\s\S]*?\1/i,greedy:!0,alias:"string"},string:{pattern:/(?:[rub]|br|rb)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,greedy:!0},function:{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)\w+/i,lookbehind:!0},decorator:{pattern:/(^[\t ]*)@\w+(?:\.\w+)*/m,lookbehind:!0,alias:["annotation","punctuation"],inside:{punctuation:/\./}},keyword:/\b(?:_(?=\s*:)|and|as|assert|async|await|break|case|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|match|nonlocal|not|or|pass|print|raise|return|try|while|with|yield)\b/,builtin:/\b(?:__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip)\b/,boolean:/\b(?:False|None|True)\b/,number:/\b0(?:b(?:_?[01])+|o(?:_?[0-7])+|x(?:_?[a-f0-9])+)\b|(?:\b\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\B\.\d+(?:_\d+)*)(?:e[+-]?\d+(?:_\d+)*)?j?(?!\w)/i,operator:/[-+%=]=?|!=|:=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,punctuation:/[{}[\];(),.:]/},r.languages.python["string-interpolation"].inside.interpolation.inside.rest=r.languages.python,r.languages.py=r.languages.python,r.languages.reason=r.languages.extend("clike",{string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/,greedy:!0},"class-name":/\b[A-Z]\w*/,keyword:/\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/,operator:/\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:asr|land|lor|lsl|lsr|lxor|mod)\b/}),r.languages.insertBefore("reason","class-name",{char:{pattern:/'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/,greedy:!0},constructor:/\b[A-Z]\w*\b(?!\s*\.)/,label:{pattern:/\b[a-z]\w*(?=::)/,alias:"symbol"}}),delete r.languages.reason.function,function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m,lookbehind:!0,greedy:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,greedy:!0,inside:{atrule:/(?:@[\w-]+|[+=])/}}}),delete e.languages.sass.atrule;var t=/\$[-\w]+|#\{\$[-\w]+\}/,n=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|not|or)\b/,{pattern:/(\s)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,greedy:!0,inside:{punctuation:/:/,variable:t,operator:n}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m,greedy:!0,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:t,operator:n,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/^([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/m,lookbehind:!0,greedy:!0}})}(r),r.languages.scss=r.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-](?:\([^()]+\)|[^()\s]|\s+(?!\s))*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)?url(?=\()/i,selector:{pattern:/(?=\S)[^@;{}()]?(?:[^@;{}()\s]|\s+(?!\s)|#\{\$[-\w]+\})+(?=\s*\{(?:\}|\s|[^}][^:{}]*[:{][^}]))/,inside:{parent:{pattern:/&/,alias:"important"},placeholder:/%[-\w]+/,variable:/\$[-\w]+|#\{\$[-\w]+\}/}},property:{pattern:/(?:[-\w]|\$[-\w]|#\{\$[-\w]+\})+(?=\s*:)/,inside:{variable:/\$[-\w]+|#\{\$[-\w]+\}/}}}),r.languages.insertBefore("scss","atrule",{keyword:[/@(?:content|debug|each|else(?: if)?|extend|for|forward|function|if|import|include|mixin|return|use|warn|while)\b/i,{pattern:/( )(?:from|through)(?= )/,lookbehind:!0}]}),r.languages.insertBefore("scss","important",{variable:/\$[-\w]+|#\{\$[-\w]+\}/}),r.languages.insertBefore("scss","function",{"module-modifier":{pattern:/\b(?:as|hide|show|with)\b/i,alias:"keyword"},placeholder:{pattern:/%[-\w]+/,alias:"selector"},statement:{pattern:/\B!(?:default|optional)\b/i,alias:"keyword"},boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"},operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|not|or)(?=\s)/,lookbehind:!0}}),r.languages.scss.atrule.inside.rest=r.languages.scss,function(e){var t={pattern:/(\b\d+)(?:%|[a-z]+)/,lookbehind:!0},n={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0},s={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},url:{pattern:/\burl\((["']?).*?\1\)/i,greedy:!0},string:{pattern:/("|')(?:(?!\1)[^\\\r\n]|\\(?:\r\n|[\s\S]))*\1/,greedy:!0},interpolation:null,func:null,important:/\B!(?:important|optional)\b/i,keyword:{pattern:/(^|\s+)(?:(?:else|for|if|return|unless)(?=\s|$)|@[\w-]+)/,lookbehind:!0},hexcode:/#[\da-f]{3,6}/i,color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:t,number:n,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:t,boolean:/\b(?:false|true)\b/,operator:[/~|[+!\/%<>?=]=?|[-:]=|\*[*=]?|\.{2,3}|&&|\|\||\B-\B|\b(?:and|in|is(?: a| defined| not|nt)?|not|or)\b/],number:n,punctuation:/[{}()\[\];:,]/};s.interpolation={pattern:/\{[^\r\n}:]+\}/,alias:"variable",inside:{delimiter:{pattern:/^\{|\}$/,alias:"punctuation"},rest:s}},s.func={pattern:/[\w-]+\([^)]*\).*/,inside:{function:/^[^(]+/,rest:s}},e.languages.stylus={"atrule-declaration":{pattern:/(^[ \t]*)@.+/m,lookbehind:!0,inside:{atrule:/^@[\w-]+/,rest:s}},"variable-declaration":{pattern:/(^[ \t]*)[\w$-]+\s*.?=[ \t]*(?:\{[^{}]*\}|\S.*|$)/m,lookbehind:!0,inside:{variable:/^\S+/,rest:s}},statement:{pattern:/(^[ \t]*)(?:else|for|if|return|unless)[ \t].+/m,lookbehind:!0,inside:{keyword:/^\S+/,rest:s}},"property-declaration":{pattern:/((?:^|\{)([ \t]*))(?:[\w-]|\{[^}\r\n]+\})+(?:\s*:\s*|[ \t]+)(?!\s)[^{\r\n]*(?:;|[^{\r\n,]$(?!(?:\r?\n|\r)(?:\{|\2[ \t])))/m,lookbehind:!0,inside:{property:{pattern:/^[^\s:]+/,inside:{interpolation:s.interpolation}},rest:s}},selector:{pattern:/(^[ \t]*)(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)(?:(?:\r?\n|\r)(?:\1(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)))*(?:,$|\{|(?=(?:\r?\n|\r)(?:\{|\1[ \t])))/m,lookbehind:!0,inside:{interpolation:s.interpolation,comment:s.comment,punctuation:/[{},]/}},func:s.func,string:s.string,comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0,greedy:!0},interpolation:s.interpolation,punctuation:/[{}()\[\];:.]/}}(r),function(e){var t=e.util.clone(e.languages.typescript);e.languages.tsx=e.languages.extend("jsx",t),delete e.languages.tsx.parameter,delete e.languages.tsx["literal-property"];var n=e.languages.tsx.tag;n.pattern=RegExp(/(^|[^\w$]|(?=<\/))/.source+"(?:"+n.pattern.source+")",n.pattern.flags),n.lookbehind=!0}(r),r.languages.wasm={comment:[/\(;[\s\S]*?;\)/,{pattern:/;;.*/,greedy:!0}],string:{pattern:/"(?:\\[\s\S]|[^"\\])*"/,greedy:!0},keyword:[{pattern:/\b(?:align|offset)=/,inside:{operator:/=/}},{pattern:/\b(?:(?:f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|neg?|nearest|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|sqrt|store(?:8|16|32)?|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))?|memory\.(?:grow|size))\b/,inside:{punctuation:/\./}},/\b(?:anyfunc|block|br(?:_if|_table)?|call(?:_indirect)?|data|drop|elem|else|end|export|func|get_(?:global|local)|global|if|import|local|loop|memory|module|mut|nop|offset|param|result|return|select|set_(?:global|local)|start|table|tee_local|then|type|unreachable)\b/],variable:/\$[\w!#$%&'*+\-./:<=>?@\\^`|~]+/,number:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/,punctuation:/[()]/};const o=r},79016:()=>{!function(e){function t(e,t){return e.replace(/<<(\d+)>>/g,(function(e,n){return"(?:"+t[+n]+")"}))}function n(e,n,s){return RegExp(t(e,n),s||"")}function s(e,t){for(var n=0;n<t;n++)e=e.replace(/<<self>>/g,(function(){return"(?:"+e+")"}));return e.replace(/<<self>>/g,"[^\\s\\S]")}var r="bool byte char decimal double dynamic float int long object sbyte short string uint ulong ushort var void",o="class enum interface record struct",a="add alias and ascending async await by descending from(?=\\s*(?:\\w|$)) get global group into init(?=\\s*;) join let nameof not notnull on or orderby partial remove select set unmanaged value when where with(?=\\s*{)",i="abstract as base break case catch checked const continue default delegate do else event explicit extern finally fixed for foreach goto if implicit in internal is lock namespace new null operator out override params private protected public readonly ref return sealed sizeof stackalloc static switch this throw try typeof unchecked unsafe using virtual volatile while yield";function c(e){return"\\b(?:"+e.trim().replace(/ /g,"|")+")\\b"}var l=c(o),d=RegExp(c(r+" "+o+" "+a+" "+i)),p=c(o+" "+a+" "+i),u=c(r+" "+o+" "+i),m=s(/<(?:[^<>;=+\-*/%&|^]|<<self>>)*>/.source,2),h=s(/\((?:[^()]|<<self>>)*\)/.source,2),g=/@?\b[A-Za-z_]\w*\b/.source,f=t(/<<0>>(?:\s*<<1>>)?/.source,[g,m]),b=t(/(?!<<0>>)<<1>>(?:\s*\.\s*<<1>>)*/.source,[p,f]),y=/\[\s*(?:,\s*)*\]/.source,x=t(/<<0>>(?:\s*(?:\?\s*)?<<1>>)*(?:\s*\?)?/.source,[b,y]),v=t(/[^,()<>[\];=+\-*/%&|^]|<<0>>|<<1>>|<<2>>/.source,[m,h,y]),A=t(/\(<<0>>+(?:,<<0>>+)+\)/.source,[v]),S=t(/(?:<<0>>|<<1>>)(?:\s*(?:\?\s*)?<<2>>)*(?:\s*\?)?/.source,[A,b,y]),E={keyword:d,punctuation:/[<>()?,.:[\]]/},L=/'(?:[^\r\n'\\]|\\.|\\[Uux][\da-fA-F]{1,8})'/.source,M=/"(?:\\.|[^\\"\r\n])*"/.source,k=/@"(?:""|\\[\s\S]|[^\\"])*"(?!")/.source;e.languages.csharp=e.languages.extend("clike",{string:[{pattern:n(/(^|[^$\\])<<0>>/.source,[k]),lookbehind:!0,greedy:!0},{pattern:n(/(^|[^@$\\])<<0>>/.source,[M]),lookbehind:!0,greedy:!0}],"class-name":[{pattern:n(/(\busing\s+static\s+)<<0>>(?=\s*;)/.source,[b]),lookbehind:!0,inside:E},{pattern:n(/(\busing\s+<<0>>\s*=\s*)<<1>>(?=\s*;)/.source,[g,S]),lookbehind:!0,inside:E},{pattern:n(/(\busing\s+)<<0>>(?=\s*=)/.source,[g]),lookbehind:!0},{pattern:n(/(\b<<0>>\s+)<<1>>/.source,[l,f]),lookbehind:!0,inside:E},{pattern:n(/(\bcatch\s*\(\s*)<<0>>/.source,[b]),lookbehind:!0,inside:E},{pattern:n(/(\bwhere\s+)<<0>>/.source,[g]),lookbehind:!0},{pattern:n(/(\b(?:is(?:\s+not)?|as)\s+)<<0>>/.source,[x]),lookbehind:!0,inside:E},{pattern:n(/\b<<0>>(?=\s+(?!<<1>>|with\s*\{)<<2>>(?:\s*[=,;:{)\]]|\s+(?:in|when)\b))/.source,[S,u,g]),inside:E}],keyword:d,number:/(?:\b0(?:x[\da-f_]*[\da-f]|b[01_]*[01])|(?:\B\.\d+(?:_+\d+)*|\b\d+(?:_+\d+)*(?:\.\d+(?:_+\d+)*)?)(?:e[-+]?\d+(?:_+\d+)*)?)(?:[dflmu]|lu|ul)?\b/i,operator:/>>=?|<<=?|[-=]>|([-+&|])\1|~|\?\?=?|[-+*/%&|^!=<>]=?/,punctuation:/\?\.?|::|[{}[\];(),.:]/}),e.languages.insertBefore("csharp","number",{range:{pattern:/\.\./,alias:"operator"}}),e.languages.insertBefore("csharp","punctuation",{"named-parameter":{pattern:n(/([(,]\s*)<<0>>(?=\s*:)/.source,[g]),lookbehind:!0,alias:"punctuation"}}),e.languages.insertBefore("csharp","class-name",{namespace:{pattern:n(/(\b(?:namespace|using)\s+)<<0>>(?:\s*\.\s*<<0>>)*(?=\s*[;{])/.source,[g]),lookbehind:!0,inside:{punctuation:/\./}},"type-expression":{pattern:n(/(\b(?:default|sizeof|typeof)\s*\(\s*(?!\s))(?:[^()\s]|\s(?!\s)|<<0>>)*(?=\s*\))/.source,[h]),lookbehind:!0,alias:"class-name",inside:E},"return-type":{pattern:n(/<<0>>(?=\s+(?:<<1>>\s*(?:=>|[({]|\.\s*this\s*\[)|this\s*\[))/.source,[S,b]),inside:E,alias:"class-name"},"constructor-invocation":{pattern:n(/(\bnew\s+)<<0>>(?=\s*[[({])/.source,[S]),lookbehind:!0,inside:E,alias:"class-name"},"generic-method":{pattern:n(/<<0>>\s*<<1>>(?=\s*\()/.source,[g,m]),inside:{function:n(/^<<0>>/.source,[g]),generic:{pattern:RegExp(m),alias:"class-name",inside:E}}},"type-list":{pattern:n(/\b((?:<<0>>\s+<<1>>|record\s+<<1>>\s*<<5>>|where\s+<<2>>)\s*:\s*)(?:<<3>>|<<4>>|<<1>>\s*<<5>>|<<6>>)(?:\s*,\s*(?:<<3>>|<<4>>|<<6>>))*(?=\s*(?:where|[{;]|=>|$))/.source,[l,f,g,S,d.source,h,/\bnew\s*\(\s*\)/.source]),lookbehind:!0,inside:{"record-arguments":{pattern:n(/(^(?!new\s*\()<<0>>\s*)<<1>>/.source,[f,h]),lookbehind:!0,greedy:!0,inside:e.languages.csharp},keyword:d,"class-name":{pattern:RegExp(S),greedy:!0,inside:E},punctuation:/[,()]/}},preprocessor:{pattern:/(^[\t ]*)#.*/m,lookbehind:!0,alias:"property",inside:{directive:{pattern:/(#)\b(?:define|elif|else|endif|endregion|error|if|line|nullable|pragma|region|undef|warning)\b/,lookbehind:!0,alias:"keyword"}}}});var _=M+"|"+L,w=t(/\/(?![*/])|\/\/[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>/.source,[_]),I=s(t(/[^"'/()]|<<0>>|\(<<self>>*\)/.source,[w]),2),C=/\b(?:assembly|event|field|method|module|param|property|return|type)\b/.source,Q=t(/<<0>>(?:\s*\(<<1>>*\))?/.source,[b,I]);e.languages.insertBefore("csharp","class-name",{attribute:{pattern:n(/((?:^|[^\s\w>)?])\s*\[\s*)(?:<<0>>\s*:\s*)?<<1>>(?:\s*,\s*<<1>>)*(?=\s*\])/.source,[C,Q]),lookbehind:!0,greedy:!0,inside:{target:{pattern:n(/^<<0>>(?=\s*:)/.source,[C]),alias:"keyword"},"attribute-arguments":{pattern:n(/\(<<0>>*\)/.source,[I]),inside:e.languages.csharp},"class-name":{pattern:RegExp(b),inside:{punctuation:/\./}},punctuation:/[:,]/}}});var R=/:[^}\r\n]+/.source,D=s(t(/[^"'/()]|<<0>>|\(<<self>>*\)/.source,[w]),2),T=t(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source,[D,R]),O=s(t(/[^"'/()]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>|\(<<self>>*\)/.source,[_]),2),P=t(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source,[O,R]);function F(t,s){return{interpolation:{pattern:n(/((?:^|[^{])(?:\{\{)*)<<0>>/.source,[t]),lookbehind:!0,inside:{"format-string":{pattern:n(/(^\{(?:(?![}:])<<0>>)*)<<1>>(?=\}$)/.source,[s,R]),lookbehind:!0,inside:{punctuation:/^:/}},punctuation:/^\{|\}$/,expression:{pattern:/[\s\S]+/,alias:"language-csharp",inside:e.languages.csharp}}},string:/[\s\S]+/}}e.languages.insertBefore("csharp","string",{"interpolation-string":[{pattern:n(/(^|[^\\])(?:\$@|@\$)"(?:""|\\[\s\S]|\{\{|<<0>>|[^\\{"])*"/.source,[T]),lookbehind:!0,greedy:!0,inside:F(T,D)},{pattern:n(/(^|[^@\\])\$"(?:\\.|\{\{|<<0>>|[^\\"{])*"/.source,[P]),lookbehind:!0,greedy:!0,inside:F(P,O)}],char:{pattern:RegExp(L),greedy:!0}}),e.languages.dotnet=e.languages.cs=e.languages.csharp}(Prism)},86862:()=>{!function(e){var t=e.languages.powershell={comment:[{pattern:/(^|[^`])<#[\s\S]*?#>/,lookbehind:!0},{pattern:/(^|[^`])#.*/,lookbehind:!0}],string:[{pattern:/"(?:`[\s\S]|[^`"])*"/,greedy:!0,inside:null},{pattern:/'(?:[^']|'')*'/,greedy:!0}],namespace:/\[[a-z](?:\[(?:\[[^\]]*\]|[^\[\]])*\]|[^\[\]])*\]/i,boolean:/\$(?:false|true)\b/i,variable:/\$\w+\b/,function:[/\b(?:Add|Approve|Assert|Backup|Block|Checkpoint|Clear|Close|Compare|Complete|Compress|Confirm|Connect|Convert|ConvertFrom|ConvertTo|Copy|Debug|Deny|Disable|Disconnect|Dismount|Edit|Enable|Enter|Exit|Expand|Export|Find|ForEach|Format|Get|Grant|Group|Hide|Import|Initialize|Install|Invoke|Join|Limit|Lock|Measure|Merge|Move|New|Open|Optimize|Out|Ping|Pop|Protect|Publish|Push|Read|Receive|Redo|Register|Remove|Rename|Repair|Request|Reset|Resize|Resolve|Restart|Restore|Resume|Revoke|Save|Search|Select|Send|Set|Show|Skip|Sort|Split|Start|Step|Stop|Submit|Suspend|Switch|Sync|Tee|Test|Trace|Unblock|Undo|Uninstall|Unlock|Unprotect|Unpublish|Unregister|Update|Use|Wait|Watch|Where|Write)-[a-z]+\b/i,/\b(?:ac|cat|chdir|clc|cli|clp|clv|compare|copy|cp|cpi|cpp|cvpa|dbp|del|diff|dir|ebp|echo|epal|epcsv|epsn|erase|fc|fl|ft|fw|gal|gbp|gc|gci|gcs|gdr|gi|gl|gm|gp|gps|group|gsv|gu|gv|gwmi|iex|ii|ipal|ipcsv|ipsn|irm|iwmi|iwr|kill|lp|ls|measure|mi|mount|move|mp|mv|nal|ndr|ni|nv|ogv|popd|ps|pushd|pwd|rbp|rd|rdr|ren|ri|rm|rmdir|rni|rnp|rp|rv|rvpa|rwmi|sal|saps|sasv|sbp|sc|select|set|shcm|si|sl|sleep|sls|sort|sp|spps|spsv|start|sv|swmi|tee|trcm|type|write)\b/i],keyword:/\b(?:Begin|Break|Catch|Class|Continue|Data|Define|Do|DynamicParam|Else|ElseIf|End|Exit|Filter|Finally|For|ForEach|From|Function|If|InlineScript|Parallel|Param|Process|Return|Sequence|Switch|Throw|Trap|Try|Until|Using|Var|While|Workflow)\b/i,operator:{pattern:/(^|\W)(?:!|-(?:b?(?:and|x?or)|as|(?:Not)?(?:Contains|In|Like|Match)|eq|ge|gt|is(?:Not)?|Join|le|lt|ne|not|Replace|sh[lr])\b|-[-=]?|\+[+=]?|[*\/%]=?)/i,lookbehind:!0},punctuation:/[|{}[\];(),.]/};t.string[0].inside={function:{pattern:/(^|[^`])\$\((?:\$\([^\r\n()]*\)|(?!\$\()[^\r\n)])*\)/,lookbehind:!0,inside:t},boolean:t.boolean,variable:t.variable}}(Prism)},55979:(e,t,n)=>{var s={"./prism-csharp":79016,"./prism-powershell":86862};function r(e){var t=o(e);return n(t)}function o(e){if(!n.o(s,e)){var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}return s[e]}r.keys=function(){return Object.keys(s)},r.resolve=o,e.exports=r,r.id=55979},92703:(e,t,n)=>{"use strict";var s=n(50414);function r(){}function o(){}o.resetWarningCache=r,e.exports=function(){function e(e,t,n,r,o,a){if(a!==s){var i=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw i.name="Invariant Violation",i}}function t(){return e}e.isRequired=e;var n={array:e,bigint:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:o,resetWarningCache:r};return n.PropTypes=n,n}},45697:(e,t,n)=>{e.exports=n(92703)()},50414:e=>{"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},64448:(e,t,n)=>{"use strict";var s=n(67294),r=n(27418),o=n(63840);function a(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n<arguments.length;n++)t+="&args[]="+encodeURIComponent(arguments[n]);return"Minified React error #"+e+"; visit "+t+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}if(!s)throw Error(a(227));function i(e,t,n,s,r,o,a,i,c){var l=Array.prototype.slice.call(arguments,3);try{t.apply(n,l)}catch(d){this.onError(d)}}var c=!1,l=null,d=!1,p=null,u={onError:function(e){c=!0,l=e}};function m(e,t,n,s,r,o,a,d,p){c=!1,l=null,i.apply(u,arguments)}var h=null,g=null,f=null;function b(e,t,n){var s=e.type||"unknown-event";e.currentTarget=f(n),function(e,t,n,s,r,o,i,u,h){if(m.apply(this,arguments),c){if(!c)throw Error(a(198));var g=l;c=!1,l=null,d||(d=!0,p=g)}}(s,t,void 0,e),e.currentTarget=null}var y=null,x={};function v(){if(y)for(var e in x){var t=x[e],n=y.indexOf(e);if(!(-1<n))throw Error(a(96,e));if(!S[n]){if(!t.extractEvents)throw Error(a(97,e));for(var s in S[n]=t,n=t.eventTypes){var r=void 0,o=n[s],i=t,c=s;if(E.hasOwnProperty(c))throw Error(a(99,c));E[c]=o;var l=o.phasedRegistrationNames;if(l){for(r in l)l.hasOwnProperty(r)&&A(l[r],i,c);r=!0}else o.registrationName?(A(o.registrationName,i,c),r=!0):r=!1;if(!r)throw Error(a(98,s,e))}}}}function A(e,t,n){if(L[e])throw Error(a(100,e));L[e]=t,M[e]=t.eventTypes[n].dependencies}var S=[],E={},L={},M={};function k(e){var t,n=!1;for(t in e)if(e.hasOwnProperty(t)){var s=e[t];if(!x.hasOwnProperty(t)||x[t]!==s){if(x[t])throw Error(a(102,t));x[t]=s,n=!0}}n&&v()}var _=!("undefined"==typeof window||void 0===window.document||void 0===window.document.createElement),w=null,I=null,C=null;function Q(e){if(e=g(e)){if("function"!=typeof w)throw Error(a(280));var t=e.stateNode;t&&(t=h(t),w(e.stateNode,e.type,t))}}function R(e){I?C?C.push(e):C=[e]:I=e}function D(){if(I){var e=I,t=C;if(C=I=null,Q(e),t)for(e=0;e<t.length;e++)Q(t[e])}}function T(e,t){return e(t)}function O(e,t,n,s,r){return e(t,n,s,r)}function P(){}var F=T,N=!1,B=!1;function V(){null===I&&null===C||(P(),D())}function G(e,t,n){if(B)return e(t,n);B=!0;try{return F(e,t,n)}finally{B=!1,V()}}var W=/^[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD][:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-.0-9\u00B7\u0300-\u036F\u203F-\u2040]*$/,U=Object.prototype.hasOwnProperty,z={},H={};function j(e,t,n,s,r,o){this.acceptsBooleans=2===t||3===t||4===t,this.attributeName=s,this.attributeNamespace=r,this.mustUseProperty=n,this.propertyName=e,this.type=t,this.sanitizeURL=o}var $={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach((function(e){$[e]=new j(e,0,!1,e,null,!1)})),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach((function(e){var t=e[0];$[t]=new j(t,1,!1,e[1],null,!1)})),["contentEditable","draggable","spellCheck","value"].forEach((function(e){$[e]=new j(e,2,!1,e.toLowerCase(),null,!1)})),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach((function(e){$[e]=new j(e,2,!1,e,null,!1)})),"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach((function(e){$[e]=new j(e,3,!1,e.toLowerCase(),null,!1)})),["checked","multiple","muted","selected"].forEach((function(e){$[e]=new j(e,3,!0,e,null,!1)})),["capture","download"].forEach((function(e){$[e]=new j(e,4,!1,e,null,!1)})),["cols","rows","size","span"].forEach((function(e){$[e]=new j(e,6,!1,e,null,!1)})),["rowSpan","start"].forEach((function(e){$[e]=new j(e,5,!1,e.toLowerCase(),null,!1)}));var Z=/[\-:]([a-z])/g;function K(e){return e[1].toUpperCase()}"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach((function(e){var t=e.replace(Z,K);$[t]=new j(t,1,!1,e,null,!1)})),"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach((function(e){var t=e.replace(Z,K);$[t]=new j(t,1,!1,e,"http://www.w3.org/1999/xlink",!1)})),["xml:base","xml:lang","xml:space"].forEach((function(e){var t=e.replace(Z,K);$[t]=new j(t,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1)})),["tabIndex","crossOrigin"].forEach((function(e){$[e]=new j(e,1,!1,e.toLowerCase(),null,!1)})),$.xlinkHref=new j("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0),["src","href","action","formAction"].forEach((function(e){$[e]=new j(e,1,!1,e.toLowerCase(),null,!0)}));var X=s.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED;function Y(e,t,n,s){var r=$.hasOwnProperty(t)?$[t]:null;(null!==r?0===r.type:!s&&(2<t.length&&("o"===t[0]||"O"===t[0])&&("n"===t[1]||"N"===t[1])))||(function(e,t,n,s){if(null==t||function(e,t,n,s){if(null!==n&&0===n.type)return!1;switch(typeof t){case"function":case"symbol":return!0;case"boolean":return!s&&(null!==n?!n.acceptsBooleans:"data-"!==(e=e.toLowerCase().slice(0,5))&&"aria-"!==e);default:return!1}}(e,t,n,s))return!0;if(s)return!1;if(null!==n)switch(n.type){case 3:return!t;case 4:return!1===t;case 5:return isNaN(t);case 6:return isNaN(t)||1>t}return!1}(t,n,r,s)&&(n=null),s||null===r?function(e){return!!U.call(H,e)||!U.call(z,e)&&(W.test(e)?H[e]=!0:(z[e]=!0,!1))}(t)&&(null===n?e.removeAttribute(t):e.setAttribute(t,""+n)):r.mustUseProperty?e[r.propertyName]=null===n?3!==r.type&&"":n:(t=r.attributeName,s=r.attributeNamespace,null===n?e.removeAttribute(t):(n=3===(r=r.type)||4===r&&!0===n?"":""+n,s?e.setAttributeNS(s,t,n):e.setAttribute(t,n))))}X.hasOwnProperty("ReactCurrentDispatcher")||(X.ReactCurrentDispatcher={current:null}),X.hasOwnProperty("ReactCurrentBatchConfig")||(X.ReactCurrentBatchConfig={suspense:null});var q=/^(.*)[\\\/]/,J="function"==typeof Symbol&&Symbol.for,ee=J?Symbol.for("react.element"):60103,te=J?Symbol.for("react.portal"):60106,ne=J?Symbol.for("react.fragment"):60107,se=J?Symbol.for("react.strict_mode"):60108,re=J?Symbol.for("react.profiler"):60114,oe=J?Symbol.for("react.provider"):60109,ae=J?Symbol.for("react.context"):60110,ie=J?Symbol.for("react.concurrent_mode"):60111,ce=J?Symbol.for("react.forward_ref"):60112,le=J?Symbol.for("react.suspense"):60113,de=J?Symbol.for("react.suspense_list"):60120,pe=J?Symbol.for("react.memo"):60115,ue=J?Symbol.for("react.lazy"):60116,me=J?Symbol.for("react.block"):60121,he="function"==typeof Symbol&&Symbol.iterator;function ge(e){return null===e||"object"!=typeof e?null:"function"==typeof(e=he&&e[he]||e["@@iterator"])?e:null}function fe(e){if(null==e)return null;if("function"==typeof e)return e.displayName||e.name||null;if("string"==typeof e)return e;switch(e){case ne:return"Fragment";case te:return"Portal";case re:return"Profiler";case se:return"StrictMode";case le:return"Suspense";case de:return"SuspenseList"}if("object"==typeof e)switch(e.$$typeof){case ae:return"Context.Consumer";case oe:return"Context.Provider";case ce:var t=e.render;return t=t.displayName||t.name||"",e.displayName||(""!==t?"ForwardRef("+t+")":"ForwardRef");case pe:return fe(e.type);case me:return fe(e.render);case ue:if(e=1===e._status?e._result:null)return fe(e)}return null}function be(e){var t="";do{e:switch(e.tag){case 3:case 4:case 6:case 7:case 10:case 9:var n="";break e;default:var s=e._debugOwner,r=e._debugSource,o=fe(e.type);n=null,s&&(n=fe(s.type)),s=o,o="",r?o=" (at "+r.fileName.replace(q,"")+":"+r.lineNumber+")":n&&(o=" (created by "+n+")"),n="\n    in "+(s||"Unknown")+o}t+=n,e=e.return}while(e);return t}function ye(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}function xe(e){var t=e.type;return(e=e.nodeName)&&"input"===e.toLowerCase()&&("checkbox"===t||"radio"===t)}function ve(e){e._valueTracker||(e._valueTracker=function(e){var t=xe(e)?"checked":"value",n=Object.getOwnPropertyDescriptor(e.constructor.prototype,t),s=""+e[t];if(!e.hasOwnProperty(t)&&void 0!==n&&"function"==typeof n.get&&"function"==typeof n.set){var r=n.get,o=n.set;return Object.defineProperty(e,t,{configurable:!0,get:function(){return r.call(this)},set:function(e){s=""+e,o.call(this,e)}}),Object.defineProperty(e,t,{enumerable:n.enumerable}),{getValue:function(){return s},setValue:function(e){s=""+e},stopTracking:function(){e._valueTracker=null,delete e[t]}}}}(e))}function Ae(e){if(!e)return!1;var t=e._valueTracker;if(!t)return!0;var n=t.getValue(),s="";return e&&(s=xe(e)?e.checked?"true":"false":e.value),(e=s)!==n&&(t.setValue(e),!0)}function Se(e,t){var n=t.checked;return r({},t,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:null!=n?n:e._wrapperState.initialChecked})}function Ee(e,t){var n=null==t.defaultValue?"":t.defaultValue,s=null!=t.checked?t.checked:t.defaultChecked;n=ye(null!=t.value?t.value:n),e._wrapperState={initialChecked:s,initialValue:n,controlled:"checkbox"===t.type||"radio"===t.type?null!=t.checked:null!=t.value}}function Le(e,t){null!=(t=t.checked)&&Y(e,"checked",t,!1)}function Me(e,t){Le(e,t);var n=ye(t.value),s=t.type;if(null!=n)"number"===s?(0===n&&""===e.value||e.value!=n)&&(e.value=""+n):e.value!==""+n&&(e.value=""+n);else if("submit"===s||"reset"===s)return void e.removeAttribute("value");t.hasOwnProperty("value")?_e(e,t.type,n):t.hasOwnProperty("defaultValue")&&_e(e,t.type,ye(t.defaultValue)),null==t.checked&&null!=t.defaultChecked&&(e.defaultChecked=!!t.defaultChecked)}function ke(e,t,n){if(t.hasOwnProperty("value")||t.hasOwnProperty("defaultValue")){var s=t.type;if(!("submit"!==s&&"reset"!==s||void 0!==t.value&&null!==t.value))return;t=""+e._wrapperState.initialValue,n||t===e.value||(e.value=t),e.defaultValue=t}""!==(n=e.name)&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,""!==n&&(e.name=n)}function _e(e,t,n){"number"===t&&e.ownerDocument.activeElement===e||(null==n?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+n&&(e.defaultValue=""+n))}function we(e,t){return e=r({children:void 0},t),(t=function(e){var t="";return s.Children.forEach(e,(function(e){null!=e&&(t+=e)})),t}(t.children))&&(e.children=t),e}function Ie(e,t,n,s){if(e=e.options,t){t={};for(var r=0;r<n.length;r++)t["$"+n[r]]=!0;for(n=0;n<e.length;n++)r=t.hasOwnProperty("$"+e[n].value),e[n].selected!==r&&(e[n].selected=r),r&&s&&(e[n].defaultSelected=!0)}else{for(n=""+ye(n),t=null,r=0;r<e.length;r++){if(e[r].value===n)return e[r].selected=!0,void(s&&(e[r].defaultSelected=!0));null!==t||e[r].disabled||(t=e[r])}null!==t&&(t.selected=!0)}}function Ce(e,t){if(null!=t.dangerouslySetInnerHTML)throw Error(a(91));return r({},t,{value:void 0,defaultValue:void 0,children:""+e._wrapperState.initialValue})}function Qe(e,t){var n=t.value;if(null==n){if(n=t.children,t=t.defaultValue,null!=n){if(null!=t)throw Error(a(92));if(Array.isArray(n)){if(!(1>=n.length))throw Error(a(93));n=n[0]}t=n}null==t&&(t=""),n=t}e._wrapperState={initialValue:ye(n)}}function Re(e,t){var n=ye(t.value),s=ye(t.defaultValue);null!=n&&((n=""+n)!==e.value&&(e.value=n),null==t.defaultValue&&e.defaultValue!==n&&(e.defaultValue=n)),null!=s&&(e.defaultValue=""+s)}function De(e){var t=e.textContent;t===e._wrapperState.initialValue&&""!==t&&null!==t&&(e.value=t)}var Te="http://www.w3.org/1999/xhtml",Oe="http://www.w3.org/2000/svg";function Pe(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}function Fe(e,t){return null==e||"http://www.w3.org/1999/xhtml"===e?Pe(t):"http://www.w3.org/2000/svg"===e&&"foreignObject"===t?"http://www.w3.org/1999/xhtml":e}var Ne,Be,Ve=(Be=function(e,t){if(e.namespaceURI!==Oe||"innerHTML"in e)e.innerHTML=t;else{for((Ne=Ne||document.createElement("div")).innerHTML="<svg>"+t.valueOf().toString()+"</svg>",t=Ne.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;t.firstChild;)e.appendChild(t.firstChild)}},"undefined"!=typeof MSApp&&MSApp.execUnsafeLocalFunction?function(e,t,n,s){MSApp.execUnsafeLocalFunction((function(){return Be(e,t)}))}:Be);function Ge(e,t){if(t){var n=e.firstChild;if(n&&n===e.lastChild&&3===n.nodeType)return void(n.nodeValue=t)}e.textContent=t}function We(e,t){var n={};return n[e.toLowerCase()]=t.toLowerCase(),n["Webkit"+e]="webkit"+t,n["Moz"+e]="moz"+t,n}var Ue={animationend:We("Animation","AnimationEnd"),animationiteration:We("Animation","AnimationIteration"),animationstart:We("Animation","AnimationStart"),transitionend:We("Transition","TransitionEnd")},ze={},He={};function je(e){if(ze[e])return ze[e];if(!Ue[e])return e;var t,n=Ue[e];for(t in n)if(n.hasOwnProperty(t)&&t in He)return ze[e]=n[t];return e}_&&(He=document.createElement("div").style,"AnimationEvent"in window||(delete Ue.animationend.animation,delete Ue.animationiteration.animation,delete Ue.animationstart.animation),"TransitionEvent"in window||delete Ue.transitionend.transition);var $e=je("animationend"),Ze=je("animationiteration"),Ke=je("animationstart"),Xe=je("transitionend"),Ye="abort canplay canplaythrough durationchange emptied encrypted ended error loadeddata loadedmetadata loadstart pause play playing progress ratechange seeked seeking stalled suspend timeupdate volumechange waiting".split(" "),qe=new("function"==typeof WeakMap?WeakMap:Map);function Je(e){var t=qe.get(e);return void 0===t&&(t=new Map,qe.set(e,t)),t}function et(e){var t=e,n=e;if(e.alternate)for(;t.return;)t=t.return;else{e=t;do{0!=(1026&(t=e).effectTag)&&(n=t.return),e=t.return}while(e)}return 3===t.tag?n:null}function tt(e){if(13===e.tag){var t=e.memoizedState;if(null===t&&(null!==(e=e.alternate)&&(t=e.memoizedState)),null!==t)return t.dehydrated}return null}function nt(e){if(et(e)!==e)throw Error(a(188))}function st(e){if(e=function(e){var t=e.alternate;if(!t){if(null===(t=et(e)))throw Error(a(188));return t!==e?null:e}for(var n=e,s=t;;){var r=n.return;if(null===r)break;var o=r.alternate;if(null===o){if(null!==(s=r.return)){n=s;continue}break}if(r.child===o.child){for(o=r.child;o;){if(o===n)return nt(r),e;if(o===s)return nt(r),t;o=o.sibling}throw Error(a(188))}if(n.return!==s.return)n=r,s=o;else{for(var i=!1,c=r.child;c;){if(c===n){i=!0,n=r,s=o;break}if(c===s){i=!0,s=r,n=o;break}c=c.sibling}if(!i){for(c=o.child;c;){if(c===n){i=!0,n=o,s=r;break}if(c===s){i=!0,s=o,n=r;break}c=c.sibling}if(!i)throw Error(a(189))}}if(n.alternate!==s)throw Error(a(190))}if(3!==n.tag)throw Error(a(188));return n.stateNode.current===n?e:t}(e),!e)return null;for(var t=e;;){if(5===t.tag||6===t.tag)return t;if(t.child)t.child.return=t,t=t.child;else{if(t===e)break;for(;!t.sibling;){if(!t.return||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}}return null}function rt(e,t){if(null==t)throw Error(a(30));return null==e?t:Array.isArray(e)?Array.isArray(t)?(e.push.apply(e,t),e):(e.push(t),e):Array.isArray(t)?[e].concat(t):[e,t]}function ot(e,t,n){Array.isArray(e)?e.forEach(t,n):e&&t.call(n,e)}var at=null;function it(e){if(e){var t=e._dispatchListeners,n=e._dispatchInstances;if(Array.isArray(t))for(var s=0;s<t.length&&!e.isPropagationStopped();s++)b(e,t[s],n[s]);else t&&b(e,t,n);e._dispatchListeners=null,e._dispatchInstances=null,e.isPersistent()||e.constructor.release(e)}}function ct(e){if(null!==e&&(at=rt(at,e)),e=at,at=null,e){if(ot(e,it),at)throw Error(a(95));if(d)throw e=p,d=!1,p=null,e}}function lt(e){return(e=e.target||e.srcElement||window).correspondingUseElement&&(e=e.correspondingUseElement),3===e.nodeType?e.parentNode:e}function dt(e){if(!_)return!1;var t=(e="on"+e)in document;return t||((t=document.createElement("div")).setAttribute(e,"return;"),t="function"==typeof t[e]),t}var pt=[];function ut(e){e.topLevelType=null,e.nativeEvent=null,e.targetInst=null,e.ancestors.length=0,10>pt.length&&pt.push(e)}function mt(e,t,n,s){if(pt.length){var r=pt.pop();return r.topLevelType=e,r.eventSystemFlags=s,r.nativeEvent=t,r.targetInst=n,r}return{topLevelType:e,eventSystemFlags:s,nativeEvent:t,targetInst:n,ancestors:[]}}function ht(e){var t=e.targetInst,n=t;do{if(!n){e.ancestors.push(n);break}var s=n;if(3===s.tag)s=s.stateNode.containerInfo;else{for(;s.return;)s=s.return;s=3!==s.tag?null:s.stateNode.containerInfo}if(!s)break;5!==(t=n.tag)&&6!==t||e.ancestors.push(n),n=Qn(s)}while(n);for(n=0;n<e.ancestors.length;n++){t=e.ancestors[n];var r=lt(e.nativeEvent);s=e.topLevelType;var o=e.nativeEvent,a=e.eventSystemFlags;0===n&&(a|=64);for(var i=null,c=0;c<S.length;c++){var l=S[c];l&&(l=l.extractEvents(s,t,o,r,a))&&(i=rt(i,l))}ct(i)}}function gt(e,t,n){if(!n.has(e)){switch(e){case"scroll":Kt(t,"scroll",!0);break;case"focus":case"blur":Kt(t,"focus",!0),Kt(t,"blur",!0),n.set("blur",null),n.set("focus",null);break;case"cancel":case"close":dt(e)&&Kt(t,e,!0);break;case"invalid":case"submit":case"reset":break;default:-1===Ye.indexOf(e)&&Zt(e,t)}n.set(e,null)}}var ft,bt,yt,xt=!1,vt=[],At=null,St=null,Et=null,Lt=new Map,Mt=new Map,kt=[],_t="mousedown mouseup touchcancel touchend touchstart auxclick dblclick pointercancel pointerdown pointerup dragend dragstart drop compositionend compositionstart keydown keypress keyup input textInput close cancel copy cut paste click change contextmenu reset submit".split(" "),wt="focus blur dragenter dragleave mouseover mouseout pointerover pointerout gotpointercapture lostpointercapture".split(" ");function It(e,t,n,s,r){return{blockedOn:e,topLevelType:t,eventSystemFlags:32|n,nativeEvent:r,container:s}}function Ct(e,t){switch(e){case"focus":case"blur":At=null;break;case"dragenter":case"dragleave":St=null;break;case"mouseover":case"mouseout":Et=null;break;case"pointerover":case"pointerout":Lt.delete(t.pointerId);break;case"gotpointercapture":case"lostpointercapture":Mt.delete(t.pointerId)}}function Qt(e,t,n,s,r,o){return null===e||e.nativeEvent!==o?(e=It(t,n,s,r,o),null!==t&&(null!==(t=Rn(t))&&bt(t)),e):(e.eventSystemFlags|=s,e)}function Rt(e){var t=Qn(e.target);if(null!==t){var n=et(t);if(null!==n)if(13===(t=n.tag)){if(null!==(t=tt(n)))return e.blockedOn=t,void o.unstable_runWithPriority(e.priority,(function(){yt(n)}))}else if(3===t&&n.stateNode.hydrate)return void(e.blockedOn=3===n.tag?n.stateNode.containerInfo:null)}e.blockedOn=null}function Dt(e){if(null!==e.blockedOn)return!1;var t=Jt(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);if(null!==t){var n=Rn(t);return null!==n&&bt(n),e.blockedOn=t,!1}return!0}function Tt(e,t,n){Dt(e)&&n.delete(t)}function Ot(){for(xt=!1;0<vt.length;){var e=vt[0];if(null!==e.blockedOn){null!==(e=Rn(e.blockedOn))&&ft(e);break}var t=Jt(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);null!==t?e.blockedOn=t:vt.shift()}null!==At&&Dt(At)&&(At=null),null!==St&&Dt(St)&&(St=null),null!==Et&&Dt(Et)&&(Et=null),Lt.forEach(Tt),Mt.forEach(Tt)}function Pt(e,t){e.blockedOn===t&&(e.blockedOn=null,xt||(xt=!0,o.unstable_scheduleCallback(o.unstable_NormalPriority,Ot)))}function Ft(e){function t(t){return Pt(t,e)}if(0<vt.length){Pt(vt[0],e);for(var n=1;n<vt.length;n++){var s=vt[n];s.blockedOn===e&&(s.blockedOn=null)}}for(null!==At&&Pt(At,e),null!==St&&Pt(St,e),null!==Et&&Pt(Et,e),Lt.forEach(t),Mt.forEach(t),n=0;n<kt.length;n++)(s=kt[n]).blockedOn===e&&(s.blockedOn=null);for(;0<kt.length&&null===(n=kt[0]).blockedOn;)Rt(n),null===n.blockedOn&&kt.shift()}var Nt={},Bt=new Map,Vt=new Map,Gt=["abort","abort",$e,"animationEnd",Ze,"animationIteration",Ke,"animationStart","canplay","canPlay","canplaythrough","canPlayThrough","durationchange","durationChange","emptied","emptied","encrypted","encrypted","ended","ended","error","error","gotpointercapture","gotPointerCapture","load","load","loadeddata","loadedData","loadedmetadata","loadedMetadata","loadstart","loadStart","lostpointercapture","lostPointerCapture","playing","playing","progress","progress","seeking","seeking","stalled","stalled","suspend","suspend","timeupdate","timeUpdate",Xe,"transitionEnd","waiting","waiting"];function Wt(e,t){for(var n=0;n<e.length;n+=2){var s=e[n],r=e[n+1],o="on"+(r[0].toUpperCase()+r.slice(1));o={phasedRegistrationNames:{bubbled:o,captured:o+"Capture"},dependencies:[s],eventPriority:t},Vt.set(s,t),Bt.set(s,o),Nt[r]=o}}Wt("blur blur cancel cancel click click close close contextmenu contextMenu copy copy cut cut auxclick auxClick dblclick doubleClick dragend dragEnd dragstart dragStart drop drop focus focus input input invalid invalid keydown keyDown keypress keyPress keyup keyUp mousedown mouseDown mouseup mouseUp paste paste pause pause play play pointercancel pointerCancel pointerdown pointerDown pointerup pointerUp ratechange rateChange reset reset seeked seeked submit submit touchcancel touchCancel touchend touchEnd touchstart touchStart volumechange volumeChange".split(" "),0),Wt("drag drag dragenter dragEnter dragexit dragExit dragleave dragLeave dragover dragOver mousemove mouseMove mouseout mouseOut mouseover mouseOver pointermove pointerMove pointerout pointerOut pointerover pointerOver scroll scroll toggle toggle touchmove touchMove wheel wheel".split(" "),1),Wt(Gt,2);for(var Ut="change selectionchange textInput compositionstart compositionend compositionupdate".split(" "),zt=0;zt<Ut.length;zt++)Vt.set(Ut[zt],0);var Ht=o.unstable_UserBlockingPriority,jt=o.unstable_runWithPriority,$t=!0;function Zt(e,t){Kt(t,e,!1)}function Kt(e,t,n){var s=Vt.get(t);switch(void 0===s?2:s){case 0:s=Xt.bind(null,t,1,e);break;case 1:s=Yt.bind(null,t,1,e);break;default:s=qt.bind(null,t,1,e)}n?e.addEventListener(t,s,!0):e.addEventListener(t,s,!1)}function Xt(e,t,n,s){N||P();var r=qt,o=N;N=!0;try{O(r,e,t,n,s)}finally{(N=o)||V()}}function Yt(e,t,n,s){jt(Ht,qt.bind(null,e,t,n,s))}function qt(e,t,n,s){if($t)if(0<vt.length&&-1<_t.indexOf(e))e=It(null,e,t,n,s),vt.push(e);else{var r=Jt(e,t,n,s);if(null===r)Ct(e,s);else if(-1<_t.indexOf(e))e=It(r,e,t,n,s),vt.push(e);else if(!function(e,t,n,s,r){switch(t){case"focus":return At=Qt(At,e,t,n,s,r),!0;case"dragenter":return St=Qt(St,e,t,n,s,r),!0;case"mouseover":return Et=Qt(Et,e,t,n,s,r),!0;case"pointerover":var o=r.pointerId;return Lt.set(o,Qt(Lt.get(o)||null,e,t,n,s,r)),!0;case"gotpointercapture":return o=r.pointerId,Mt.set(o,Qt(Mt.get(o)||null,e,t,n,s,r)),!0}return!1}(r,e,t,n,s)){Ct(e,s),e=mt(e,s,null,t);try{G(ht,e)}finally{ut(e)}}}}function Jt(e,t,n,s){if(null!==(n=Qn(n=lt(s)))){var r=et(n);if(null===r)n=null;else{var o=r.tag;if(13===o){if(null!==(n=tt(r)))return n;n=null}else if(3===o){if(r.stateNode.hydrate)return 3===r.tag?r.stateNode.containerInfo:null;n=null}else r!==n&&(n=null)}}e=mt(e,s,n,t);try{G(ht,e)}finally{ut(e)}return null}var en={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},tn=["Webkit","ms","Moz","O"];function nn(e,t,n){return null==t||"boolean"==typeof t||""===t?"":n||"number"!=typeof t||0===t||en.hasOwnProperty(e)&&en[e]?(""+t).trim():t+"px"}function sn(e,t){for(var n in e=e.style,t)if(t.hasOwnProperty(n)){var s=0===n.indexOf("--"),r=nn(n,t[n],s);"float"===n&&(n="cssFloat"),s?e.setProperty(n,r):e[n]=r}}Object.keys(en).forEach((function(e){tn.forEach((function(t){t=t+e.charAt(0).toUpperCase()+e.substring(1),en[t]=en[e]}))}));var rn=r({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function on(e,t){if(t){if(rn[e]&&(null!=t.children||null!=t.dangerouslySetInnerHTML))throw Error(a(137,e,""));if(null!=t.dangerouslySetInnerHTML){if(null!=t.children)throw Error(a(60));if("object"!=typeof t.dangerouslySetInnerHTML||!("__html"in t.dangerouslySetInnerHTML))throw Error(a(61))}if(null!=t.style&&"object"!=typeof t.style)throw Error(a(62,""))}}function an(e,t){if(-1===e.indexOf("-"))return"string"==typeof t.is;switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}var cn=Te;function ln(e,t){var n=Je(e=9===e.nodeType||11===e.nodeType?e:e.ownerDocument);t=M[t];for(var s=0;s<t.length;s++)gt(t[s],e,n)}function dn(){}function pn(e){if(void 0===(e=e||("undefined"!=typeof document?document:void 0)))return null;try{return e.activeElement||e.body}catch(t){return e.body}}function un(e){for(;e&&e.firstChild;)e=e.firstChild;return e}function mn(e,t){var n,s=un(e);for(e=0;s;){if(3===s.nodeType){if(n=e+s.textContent.length,e<=t&&n>=t)return{node:s,offset:t-e};e=n}e:{for(;s;){if(s.nextSibling){s=s.nextSibling;break e}s=s.parentNode}s=void 0}s=un(s)}}function hn(e,t){return!(!e||!t)&&(e===t||(!e||3!==e.nodeType)&&(t&&3===t.nodeType?hn(e,t.parentNode):"contains"in e?e.contains(t):!!e.compareDocumentPosition&&!!(16&e.compareDocumentPosition(t))))}function gn(){for(var e=window,t=pn();t instanceof e.HTMLIFrameElement;){try{var n="string"==typeof t.contentWindow.location.href}catch(s){n=!1}if(!n)break;t=pn((e=t.contentWindow).document)}return t}function fn(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return t&&("input"===t&&("text"===e.type||"search"===e.type||"tel"===e.type||"url"===e.type||"password"===e.type)||"textarea"===t||"true"===e.contentEditable)}var bn="$?",yn="$!",xn=null,vn=null;function An(e,t){switch(e){case"button":case"input":case"select":case"textarea":return!!t.autoFocus}return!1}function Sn(e,t){return"textarea"===e||"option"===e||"noscript"===e||"string"==typeof t.children||"number"==typeof t.children||"object"==typeof t.dangerouslySetInnerHTML&&null!==t.dangerouslySetInnerHTML&&null!=t.dangerouslySetInnerHTML.__html}var En="function"==typeof setTimeout?setTimeout:void 0,Ln="function"==typeof clearTimeout?clearTimeout:void 0;function Mn(e){for(;null!=e;e=e.nextSibling){var t=e.nodeType;if(1===t||3===t)break}return e}function kn(e){e=e.previousSibling;for(var t=0;e;){if(8===e.nodeType){var n=e.data;if("$"===n||n===yn||n===bn){if(0===t)return e;t--}else"/$"===n&&t++}e=e.previousSibling}return null}var _n=Math.random().toString(36).slice(2),wn="__reactInternalInstance$"+_n,In="__reactEventHandlers$"+_n,Cn="__reactContainere$"+_n;function Qn(e){var t=e[wn];if(t)return t;for(var n=e.parentNode;n;){if(t=n[Cn]||n[wn]){if(n=t.alternate,null!==t.child||null!==n&&null!==n.child)for(e=kn(e);null!==e;){if(n=e[wn])return n;e=kn(e)}return t}n=(e=n).parentNode}return null}function Rn(e){return!(e=e[wn]||e[Cn])||5!==e.tag&&6!==e.tag&&13!==e.tag&&3!==e.tag?null:e}function Dn(e){if(5===e.tag||6===e.tag)return e.stateNode;throw Error(a(33))}function Tn(e){return e[In]||null}function On(e){do{e=e.return}while(e&&5!==e.tag);return e||null}function Pn(e,t){var n=e.stateNode;if(!n)return null;var s=h(n);if(!s)return null;n=s[t];e:switch(t){case"onClick":case"onClickCapture":case"onDoubleClick":case"onDoubleClickCapture":case"onMouseDown":case"onMouseDownCapture":case"onMouseMove":case"onMouseMoveCapture":case"onMouseUp":case"onMouseUpCapture":case"onMouseEnter":(s=!s.disabled)||(s=!("button"===(e=e.type)||"input"===e||"select"===e||"textarea"===e)),e=!s;break e;default:e=!1}if(e)return null;if(n&&"function"!=typeof n)throw Error(a(231,t,typeof n));return n}function Fn(e,t,n){(t=Pn(e,n.dispatchConfig.phasedRegistrationNames[t]))&&(n._dispatchListeners=rt(n._dispatchListeners,t),n._dispatchInstances=rt(n._dispatchInstances,e))}function Nn(e){if(e&&e.dispatchConfig.phasedRegistrationNames){for(var t=e._targetInst,n=[];t;)n.push(t),t=On(t);for(t=n.length;0<t--;)Fn(n[t],"captured",e);for(t=0;t<n.length;t++)Fn(n[t],"bubbled",e)}}function Bn(e,t,n){e&&n&&n.dispatchConfig.registrationName&&(t=Pn(e,n.dispatchConfig.registrationName))&&(n._dispatchListeners=rt(n._dispatchListeners,t),n._dispatchInstances=rt(n._dispatchInstances,e))}function Vn(e){e&&e.dispatchConfig.registrationName&&Bn(e._targetInst,null,e)}function Gn(e){ot(e,Nn)}var Wn=null,Un=null,zn=null;function Hn(){if(zn)return zn;var e,t,n=Un,s=n.length,r="value"in Wn?Wn.value:Wn.textContent,o=r.length;for(e=0;e<s&&n[e]===r[e];e++);var a=s-e;for(t=1;t<=a&&n[s-t]===r[o-t];t++);return zn=r.slice(e,1<t?1-t:void 0)}function jn(){return!0}function $n(){return!1}function Zn(e,t,n,s){for(var r in this.dispatchConfig=e,this._targetInst=t,this.nativeEvent=n,e=this.constructor.Interface)e.hasOwnProperty(r)&&((t=e[r])?this[r]=t(n):"target"===r?this.target=s:this[r]=n[r]);return this.isDefaultPrevented=(null!=n.defaultPrevented?n.defaultPrevented:!1===n.returnValue)?jn:$n,this.isPropagationStopped=$n,this}function Kn(e,t,n,s){if(this.eventPool.length){var r=this.eventPool.pop();return this.call(r,e,t,n,s),r}return new this(e,t,n,s)}function Xn(e){if(!(e instanceof this))throw Error(a(279));e.destructor(),10>this.eventPool.length&&this.eventPool.push(e)}function Yn(e){e.eventPool=[],e.getPooled=Kn,e.release=Xn}r(Zn.prototype,{preventDefault:function(){this.defaultPrevented=!0;var e=this.nativeEvent;e&&(e.preventDefault?e.preventDefault():"unknown"!=typeof e.returnValue&&(e.returnValue=!1),this.isDefaultPrevented=jn)},stopPropagation:function(){var e=this.nativeEvent;e&&(e.stopPropagation?e.stopPropagation():"unknown"!=typeof e.cancelBubble&&(e.cancelBubble=!0),this.isPropagationStopped=jn)},persist:function(){this.isPersistent=jn},isPersistent:$n,destructor:function(){var e,t=this.constructor.Interface;for(e in t)this[e]=null;this.nativeEvent=this._targetInst=this.dispatchConfig=null,this.isPropagationStopped=this.isDefaultPrevented=$n,this._dispatchInstances=this._dispatchListeners=null}}),Zn.Interface={type:null,target:null,currentTarget:function(){return null},eventPhase:null,bubbles:null,cancelable:null,timeStamp:function(e){return e.timeStamp||Date.now()},defaultPrevented:null,isTrusted:null},Zn.extend=function(e){function t(){}function n(){return s.apply(this,arguments)}var s=this;t.prototype=s.prototype;var o=new t;return r(o,n.prototype),n.prototype=o,n.prototype.constructor=n,n.Interface=r({},s.Interface,e),n.extend=s.extend,Yn(n),n},Yn(Zn);var qn=Zn.extend({data:null}),Jn=Zn.extend({data:null}),es=[9,13,27,32],ts=_&&"CompositionEvent"in window,ns=null;_&&"documentMode"in document&&(ns=document.documentMode);var ss=_&&"TextEvent"in window&&!ns,rs=_&&(!ts||ns&&8<ns&&11>=ns),os=String.fromCharCode(32),as={beforeInput:{phasedRegistrationNames:{bubbled:"onBeforeInput",captured:"onBeforeInputCapture"},dependencies:["compositionend","keypress","textInput","paste"]},compositionEnd:{phasedRegistrationNames:{bubbled:"onCompositionEnd",captured:"onCompositionEndCapture"},dependencies:"blur compositionend keydown keypress keyup mousedown".split(" ")},compositionStart:{phasedRegistrationNames:{bubbled:"onCompositionStart",captured:"onCompositionStartCapture"},dependencies:"blur compositionstart keydown keypress keyup mousedown".split(" ")},compositionUpdate:{phasedRegistrationNames:{bubbled:"onCompositionUpdate",captured:"onCompositionUpdateCapture"},dependencies:"blur compositionupdate keydown keypress keyup mousedown".split(" ")}},is=!1;function cs(e,t){switch(e){case"keyup":return-1!==es.indexOf(t.keyCode);case"keydown":return 229!==t.keyCode;case"keypress":case"mousedown":case"blur":return!0;default:return!1}}function ls(e){return"object"==typeof(e=e.detail)&&"data"in e?e.data:null}var ds=!1;var ps={eventTypes:as,extractEvents:function(e,t,n,s){var r;if(ts)e:{switch(e){case"compositionstart":var o=as.compositionStart;break e;case"compositionend":o=as.compositionEnd;break e;case"compositionupdate":o=as.compositionUpdate;break e}o=void 0}else ds?cs(e,n)&&(o=as.compositionEnd):"keydown"===e&&229===n.keyCode&&(o=as.compositionStart);return o?(rs&&"ko"!==n.locale&&(ds||o!==as.compositionStart?o===as.compositionEnd&&ds&&(r=Hn()):(Un="value"in(Wn=s)?Wn.value:Wn.textContent,ds=!0)),o=qn.getPooled(o,t,n,s),r?o.data=r:null!==(r=ls(n))&&(o.data=r),Gn(o),r=o):r=null,(e=ss?function(e,t){switch(e){case"compositionend":return ls(t);case"keypress":return 32!==t.which?null:(is=!0,os);case"textInput":return(e=t.data)===os&&is?null:e;default:return null}}(e,n):function(e,t){if(ds)return"compositionend"===e||!ts&&cs(e,t)?(e=Hn(),zn=Un=Wn=null,ds=!1,e):null;switch(e){case"paste":default:return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1<t.char.length)return t.char;if(t.which)return String.fromCharCode(t.which)}return null;case"compositionend":return rs&&"ko"!==t.locale?null:t.data}}(e,n))?((t=Jn.getPooled(as.beforeInput,t,n,s)).data=e,Gn(t)):t=null,null===r?t:null===t?r:[r,t]}},us={color:!0,date:!0,datetime:!0,"datetime-local":!0,email:!0,month:!0,number:!0,password:!0,range:!0,search:!0,tel:!0,text:!0,time:!0,url:!0,week:!0};function ms(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return"input"===t?!!us[e.type]:"textarea"===t}var hs={change:{phasedRegistrationNames:{bubbled:"onChange",captured:"onChangeCapture"},dependencies:"blur change click focus input keydown keyup selectionchange".split(" ")}};function gs(e,t,n){return(e=Zn.getPooled(hs.change,e,t,n)).type="change",R(n),Gn(e),e}var fs=null,bs=null;function ys(e){ct(e)}function xs(e){if(Ae(Dn(e)))return e}function vs(e,t){if("change"===e)return t}var As=!1;function Ss(){fs&&(fs.detachEvent("onpropertychange",Es),bs=fs=null)}function Es(e){if("value"===e.propertyName&&xs(bs))if(e=gs(bs,e,lt(e)),N)ct(e);else{N=!0;try{T(ys,e)}finally{N=!1,V()}}}function Ls(e,t,n){"focus"===e?(Ss(),bs=n,(fs=t).attachEvent("onpropertychange",Es)):"blur"===e&&Ss()}function Ms(e){if("selectionchange"===e||"keyup"===e||"keydown"===e)return xs(bs)}function ks(e,t){if("click"===e)return xs(t)}function _s(e,t){if("input"===e||"change"===e)return xs(t)}_&&(As=dt("input")&&(!document.documentMode||9<document.documentMode));var ws={eventTypes:hs,_isInputEventSupported:As,extractEvents:function(e,t,n,s){var r=t?Dn(t):window,o=r.nodeName&&r.nodeName.toLowerCase();if("select"===o||"input"===o&&"file"===r.type)var a=vs;else if(ms(r))if(As)a=_s;else{a=Ms;var i=Ls}else(o=r.nodeName)&&"input"===o.toLowerCase()&&("checkbox"===r.type||"radio"===r.type)&&(a=ks);if(a&&(a=a(e,t)))return gs(a,n,s);i&&i(e,r,t),"blur"===e&&(e=r._wrapperState)&&e.controlled&&"number"===r.type&&_e(r,"number",r.value)}},Is=Zn.extend({view:null,detail:null}),Cs={Alt:"altKey",Control:"ctrlKey",Meta:"metaKey",Shift:"shiftKey"};function Qs(e){var t=this.nativeEvent;return t.getModifierState?t.getModifierState(e):!!(e=Cs[e])&&!!t[e]}function Rs(){return Qs}var Ds=0,Ts=0,Os=!1,Ps=!1,Fs=Is.extend({screenX:null,screenY:null,clientX:null,clientY:null,pageX:null,pageY:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,getModifierState:Rs,button:null,buttons:null,relatedTarget:function(e){return e.relatedTarget||(e.fromElement===e.srcElement?e.toElement:e.fromElement)},movementX:function(e){if("movementX"in e)return e.movementX;var t=Ds;return Ds=e.screenX,Os?"mousemove"===e.type?e.screenX-t:0:(Os=!0,0)},movementY:function(e){if("movementY"in e)return e.movementY;var t=Ts;return Ts=e.screenY,Ps?"mousemove"===e.type?e.screenY-t:0:(Ps=!0,0)}}),Ns=Fs.extend({pointerId:null,width:null,height:null,pressure:null,tangentialPressure:null,tiltX:null,tiltY:null,twist:null,pointerType:null,isPrimary:null}),Bs={mouseEnter:{registrationName:"onMouseEnter",dependencies:["mouseout","mouseover"]},mouseLeave:{registrationName:"onMouseLeave",dependencies:["mouseout","mouseover"]},pointerEnter:{registrationName:"onPointerEnter",dependencies:["pointerout","pointerover"]},pointerLeave:{registrationName:"onPointerLeave",dependencies:["pointerout","pointerover"]}},Vs={eventTypes:Bs,extractEvents:function(e,t,n,s,r){var o="mouseover"===e||"pointerover"===e,a="mouseout"===e||"pointerout"===e;if(o&&0==(32&r)&&(n.relatedTarget||n.fromElement)||!a&&!o)return null;(o=s.window===s?s:(o=s.ownerDocument)?o.defaultView||o.parentWindow:window,a)?(a=t,null!==(t=(t=n.relatedTarget||n.toElement)?Qn(t):null)&&(t!==et(t)||5!==t.tag&&6!==t.tag)&&(t=null)):a=null;if(a===t)return null;if("mouseout"===e||"mouseover"===e)var i=Fs,c=Bs.mouseLeave,l=Bs.mouseEnter,d="mouse";else"pointerout"!==e&&"pointerover"!==e||(i=Ns,c=Bs.pointerLeave,l=Bs.pointerEnter,d="pointer");if(e=null==a?o:Dn(a),o=null==t?o:Dn(t),(c=i.getPooled(c,a,n,s)).type=d+"leave",c.target=e,c.relatedTarget=o,(n=i.getPooled(l,t,n,s)).type=d+"enter",n.target=o,n.relatedTarget=e,d=t,(s=a)&&d)e:{for(l=d,a=0,e=i=s;e;e=On(e))a++;for(e=0,t=l;t;t=On(t))e++;for(;0<a-e;)i=On(i),a--;for(;0<e-a;)l=On(l),e--;for(;a--;){if(i===l||i===l.alternate)break e;i=On(i),l=On(l)}i=null}else i=null;for(l=i,i=[];s&&s!==l&&(null===(a=s.alternate)||a!==l);)i.push(s),s=On(s);for(s=[];d&&d!==l&&(null===(a=d.alternate)||a!==l);)s.push(d),d=On(d);for(d=0;d<i.length;d++)Bn(i[d],"bubbled",c);for(d=s.length;0<d--;)Bn(s[d],"captured",n);return 0==(64&r)?[c]:[c,n]}};var Gs="function"==typeof Object.is?Object.is:function(e,t){return e===t&&(0!==e||1/e==1/t)||e!=e&&t!=t},Ws=Object.prototype.hasOwnProperty;function Us(e,t){if(Gs(e,t))return!0;if("object"!=typeof e||null===e||"object"!=typeof t||null===t)return!1;var n=Object.keys(e),s=Object.keys(t);if(n.length!==s.length)return!1;for(s=0;s<n.length;s++)if(!Ws.call(t,n[s])||!Gs(e[n[s]],t[n[s]]))return!1;return!0}var zs=_&&"documentMode"in document&&11>=document.documentMode,Hs={select:{phasedRegistrationNames:{bubbled:"onSelect",captured:"onSelectCapture"},dependencies:"blur contextmenu dragend focus keydown keyup mousedown mouseup selectionchange".split(" ")}},js=null,$s=null,Zs=null,Ks=!1;function Xs(e,t){var n=t.window===t?t.document:9===t.nodeType?t:t.ownerDocument;return Ks||null==js||js!==pn(n)?null:("selectionStart"in(n=js)&&fn(n)?n={start:n.selectionStart,end:n.selectionEnd}:n={anchorNode:(n=(n.ownerDocument&&n.ownerDocument.defaultView||window).getSelection()).anchorNode,anchorOffset:n.anchorOffset,focusNode:n.focusNode,focusOffset:n.focusOffset},Zs&&Us(Zs,n)?null:(Zs=n,(e=Zn.getPooled(Hs.select,$s,e,t)).type="select",e.target=js,Gn(e),e))}var Ys={eventTypes:Hs,extractEvents:function(e,t,n,s,r,o){if(!(o=!(r=o||(s.window===s?s.document:9===s.nodeType?s:s.ownerDocument)))){e:{r=Je(r),o=M.onSelect;for(var a=0;a<o.length;a++)if(!r.has(o[a])){r=!1;break e}r=!0}o=!r}if(o)return null;switch(r=t?Dn(t):window,e){case"focus":(ms(r)||"true"===r.contentEditable)&&(js=r,$s=t,Zs=null);break;case"blur":Zs=$s=js=null;break;case"mousedown":Ks=!0;break;case"contextmenu":case"mouseup":case"dragend":return Ks=!1,Xs(n,s);case"selectionchange":if(zs)break;case"keydown":case"keyup":return Xs(n,s)}return null}},qs=Zn.extend({animationName:null,elapsedTime:null,pseudoElement:null}),Js=Zn.extend({clipboardData:function(e){return"clipboardData"in e?e.clipboardData:window.clipboardData}}),er=Is.extend({relatedTarget:null});function tr(e){var t=e.keyCode;return"charCode"in e?0===(e=e.charCode)&&13===t&&(e=13):e=t,10===e&&(e=13),32<=e||13===e?e:0}var nr={Esc:"Escape",Spacebar:" ",Left:"ArrowLeft",Up:"ArrowUp",Right:"ArrowRight",Down:"ArrowDown",Del:"Delete",Win:"OS",Menu:"ContextMenu",Apps:"ContextMenu",Scroll:"ScrollLock",MozPrintableKey:"Unidentified"},sr={8:"Backspace",9:"Tab",12:"Clear",13:"Enter",16:"Shift",17:"Control",18:"Alt",19:"Pause",20:"CapsLock",27:"Escape",32:" ",33:"PageUp",34:"PageDown",35:"End",36:"Home",37:"ArrowLeft",38:"ArrowUp",39:"ArrowRight",40:"ArrowDown",45:"Insert",46:"Delete",112:"F1",113:"F2",114:"F3",115:"F4",116:"F5",117:"F6",118:"F7",119:"F8",120:"F9",121:"F10",122:"F11",123:"F12",144:"NumLock",145:"ScrollLock",224:"Meta"},rr=Is.extend({key:function(e){if(e.key){var t=nr[e.key]||e.key;if("Unidentified"!==t)return t}return"keypress"===e.type?13===(e=tr(e))?"Enter":String.fromCharCode(e):"keydown"===e.type||"keyup"===e.type?sr[e.keyCode]||"Unidentified":""},location:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,repeat:null,locale:null,getModifierState:Rs,charCode:function(e){return"keypress"===e.type?tr(e):0},keyCode:function(e){return"keydown"===e.type||"keyup"===e.type?e.keyCode:0},which:function(e){return"keypress"===e.type?tr(e):"keydown"===e.type||"keyup"===e.type?e.keyCode:0}}),or=Fs.extend({dataTransfer:null}),ar=Is.extend({touches:null,targetTouches:null,changedTouches:null,altKey:null,metaKey:null,ctrlKey:null,shiftKey:null,getModifierState:Rs}),ir=Zn.extend({propertyName:null,elapsedTime:null,pseudoElement:null}),cr=Fs.extend({deltaX:function(e){return"deltaX"in e?e.deltaX:"wheelDeltaX"in e?-e.wheelDeltaX:0},deltaY:function(e){return"deltaY"in e?e.deltaY:"wheelDeltaY"in e?-e.wheelDeltaY:"wheelDelta"in e?-e.wheelDelta:0},deltaZ:null,deltaMode:null}),lr={eventTypes:Nt,extractEvents:function(e,t,n,s){var r=Bt.get(e);if(!r)return null;switch(e){case"keypress":if(0===tr(n))return null;case"keydown":case"keyup":e=rr;break;case"blur":case"focus":e=er;break;case"click":if(2===n.button)return null;case"auxclick":case"dblclick":case"mousedown":case"mousemove":case"mouseup":case"mouseout":case"mouseover":case"contextmenu":e=Fs;break;case"drag":case"dragend":case"dragenter":case"dragexit":case"dragleave":case"dragover":case"dragstart":case"drop":e=or;break;case"touchcancel":case"touchend":case"touchmove":case"touchstart":e=ar;break;case $e:case Ze:case Ke:e=qs;break;case Xe:e=ir;break;case"scroll":e=Is;break;case"wheel":e=cr;break;case"copy":case"cut":case"paste":e=Js;break;case"gotpointercapture":case"lostpointercapture":case"pointercancel":case"pointerdown":case"pointermove":case"pointerout":case"pointerover":case"pointerup":e=Ns;break;default:e=Zn}return Gn(t=e.getPooled(r,t,n,s)),t}};if(y)throw Error(a(101));y=Array.prototype.slice.call("ResponderEventPlugin SimpleEventPlugin EnterLeaveEventPlugin ChangeEventPlugin SelectEventPlugin BeforeInputEventPlugin".split(" ")),v(),h=Tn,g=Rn,f=Dn,k({SimpleEventPlugin:lr,EnterLeaveEventPlugin:Vs,ChangeEventPlugin:ws,SelectEventPlugin:Ys,BeforeInputEventPlugin:ps});var dr=[],pr=-1;function ur(e){0>pr||(e.current=dr[pr],dr[pr]=null,pr--)}function mr(e,t){pr++,dr[pr]=e.current,e.current=t}var hr={},gr={current:hr},fr={current:!1},br=hr;function yr(e,t){var n=e.type.contextTypes;if(!n)return hr;var s=e.stateNode;if(s&&s.__reactInternalMemoizedUnmaskedChildContext===t)return s.__reactInternalMemoizedMaskedChildContext;var r,o={};for(r in n)o[r]=t[r];return s&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=t,e.__reactInternalMemoizedMaskedChildContext=o),o}function xr(e){return null!=(e=e.childContextTypes)}function vr(){ur(fr),ur(gr)}function Ar(e,t,n){if(gr.current!==hr)throw Error(a(168));mr(gr,t),mr(fr,n)}function Sr(e,t,n){var s=e.stateNode;if(e=t.childContextTypes,"function"!=typeof s.getChildContext)return n;for(var o in s=s.getChildContext())if(!(o in e))throw Error(a(108,fe(t)||"Unknown",o));return r({},n,{},s)}function Er(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||hr,br=gr.current,mr(gr,e),mr(fr,fr.current),!0}function Lr(e,t,n){var s=e.stateNode;if(!s)throw Error(a(169));n?(e=Sr(e,t,br),s.__reactInternalMemoizedMergedChildContext=e,ur(fr),ur(gr),mr(gr,e)):ur(fr),mr(fr,n)}var Mr=o.unstable_runWithPriority,kr=o.unstable_scheduleCallback,_r=o.unstable_cancelCallback,wr=o.unstable_requestPaint,Ir=o.unstable_now,Cr=o.unstable_getCurrentPriorityLevel,Qr=o.unstable_ImmediatePriority,Rr=o.unstable_UserBlockingPriority,Dr=o.unstable_NormalPriority,Tr=o.unstable_LowPriority,Or=o.unstable_IdlePriority,Pr={},Fr=o.unstable_shouldYield,Nr=void 0!==wr?wr:function(){},Br=null,Vr=null,Gr=!1,Wr=Ir(),Ur=1e4>Wr?Ir:function(){return Ir()-Wr};function zr(){switch(Cr()){case Qr:return 99;case Rr:return 98;case Dr:return 97;case Tr:return 96;case Or:return 95;default:throw Error(a(332))}}function Hr(e){switch(e){case 99:return Qr;case 98:return Rr;case 97:return Dr;case 96:return Tr;case 95:return Or;default:throw Error(a(332))}}function jr(e,t){return e=Hr(e),Mr(e,t)}function $r(e,t,n){return e=Hr(e),kr(e,t,n)}function Zr(e){return null===Br?(Br=[e],Vr=kr(Qr,Xr)):Br.push(e),Pr}function Kr(){if(null!==Vr){var e=Vr;Vr=null,_r(e)}Xr()}function Xr(){if(!Gr&&null!==Br){Gr=!0;var e=0;try{var t=Br;jr(99,(function(){for(;e<t.length;e++){var n=t[e];do{n=n(!0)}while(null!==n)}})),Br=null}catch(n){throw null!==Br&&(Br=Br.slice(e+1)),kr(Qr,Kr),n}finally{Gr=!1}}}function Yr(e,t,n){return 1073741821-(1+((1073741821-e+t/10)/(n/=10)|0))*n}function qr(e,t){if(e&&e.defaultProps)for(var n in t=r({},t),e=e.defaultProps)void 0===t[n]&&(t[n]=e[n]);return t}var Jr={current:null},eo=null,to=null,no=null;function so(){no=to=eo=null}function ro(e){var t=Jr.current;ur(Jr),e.type._context._currentValue=t}function oo(e,t){for(;null!==e;){var n=e.alternate;if(e.childExpirationTime<t)e.childExpirationTime=t,null!==n&&n.childExpirationTime<t&&(n.childExpirationTime=t);else{if(!(null!==n&&n.childExpirationTime<t))break;n.childExpirationTime=t}e=e.return}}function ao(e,t){eo=e,no=to=null,null!==(e=e.dependencies)&&null!==e.firstContext&&(e.expirationTime>=t&&(Ta=!0),e.firstContext=null)}function io(e,t){if(no!==e&&!1!==t&&0!==t)if("number"==typeof t&&1073741823!==t||(no=e,t=1073741823),t={context:e,observedBits:t,next:null},null===to){if(null===eo)throw Error(a(308));to=t,eo.dependencies={expirationTime:0,firstContext:t,responders:null}}else to=to.next=t;return e._currentValue}var co=!1;function lo(e){e.updateQueue={baseState:e.memoizedState,baseQueue:null,shared:{pending:null},effects:null}}function po(e,t){e=e.updateQueue,t.updateQueue===e&&(t.updateQueue={baseState:e.baseState,baseQueue:e.baseQueue,shared:e.shared,effects:e.effects})}function uo(e,t){return(e={expirationTime:e,suspenseConfig:t,tag:0,payload:null,callback:null,next:null}).next=e}function mo(e,t){if(null!==(e=e.updateQueue)){var n=(e=e.shared).pending;null===n?t.next=t:(t.next=n.next,n.next=t),e.pending=t}}function ho(e,t){var n=e.alternate;null!==n&&po(n,e),null===(n=(e=e.updateQueue).baseQueue)?(e.baseQueue=t.next=t,t.next=t):(t.next=n.next,n.next=t)}function go(e,t,n,s){var o=e.updateQueue;co=!1;var a=o.baseQueue,i=o.shared.pending;if(null!==i){if(null!==a){var c=a.next;a.next=i.next,i.next=c}a=i,o.shared.pending=null,null!==(c=e.alternate)&&(null!==(c=c.updateQueue)&&(c.baseQueue=i))}if(null!==a){c=a.next;var l=o.baseState,d=0,p=null,u=null,m=null;if(null!==c)for(var h=c;;){if((i=h.expirationTime)<s){var g={expirationTime:h.expirationTime,suspenseConfig:h.suspenseConfig,tag:h.tag,payload:h.payload,callback:h.callback,next:null};null===m?(u=m=g,p=l):m=m.next=g,i>d&&(d=i)}else{null!==m&&(m=m.next={expirationTime:1073741823,suspenseConfig:h.suspenseConfig,tag:h.tag,payload:h.payload,callback:h.callback,next:null}),mc(i,h.suspenseConfig);e:{var f=e,b=h;switch(i=t,g=n,b.tag){case 1:if("function"==typeof(f=b.payload)){l=f.call(g,l,i);break e}l=f;break e;case 3:f.effectTag=-4097&f.effectTag|64;case 0:if(null==(i="function"==typeof(f=b.payload)?f.call(g,l,i):f))break e;l=r({},l,i);break e;case 2:co=!0}}null!==h.callback&&(e.effectTag|=32,null===(i=o.effects)?o.effects=[h]:i.push(h))}if(null===(h=h.next)||h===c){if(null===(i=o.shared.pending))break;h=a.next=i.next,i.next=c,o.baseQueue=a=i,o.shared.pending=null}}null===m?p=l:m.next=u,o.baseState=p,o.baseQueue=m,hc(d),e.expirationTime=d,e.memoizedState=l}}function fo(e,t,n){if(e=t.effects,t.effects=null,null!==e)for(t=0;t<e.length;t++){var s=e[t],r=s.callback;if(null!==r){if(s.callback=null,s=r,r=n,"function"!=typeof s)throw Error(a(191,s));s.call(r)}}}var bo=X.ReactCurrentBatchConfig,yo=(new s.Component).refs;function xo(e,t,n,s){n=null==(n=n(s,t=e.memoizedState))?t:r({},t,n),e.memoizedState=n,0===e.expirationTime&&(e.updateQueue.baseState=n)}var vo={isMounted:function(e){return!!(e=e._reactInternalFiber)&&et(e)===e},enqueueSetState:function(e,t,n){e=e._reactInternalFiber;var s=ec(),r=bo.suspense;(r=uo(s=tc(s,e,r),r)).payload=t,null!=n&&(r.callback=n),mo(e,r),nc(e,s)},enqueueReplaceState:function(e,t,n){e=e._reactInternalFiber;var s=ec(),r=bo.suspense;(r=uo(s=tc(s,e,r),r)).tag=1,r.payload=t,null!=n&&(r.callback=n),mo(e,r),nc(e,s)},enqueueForceUpdate:function(e,t){e=e._reactInternalFiber;var n=ec(),s=bo.suspense;(s=uo(n=tc(n,e,s),s)).tag=2,null!=t&&(s.callback=t),mo(e,s),nc(e,n)}};function Ao(e,t,n,s,r,o,a){return"function"==typeof(e=e.stateNode).shouldComponentUpdate?e.shouldComponentUpdate(s,o,a):!t.prototype||!t.prototype.isPureReactComponent||(!Us(n,s)||!Us(r,o))}function So(e,t,n){var s=!1,r=hr,o=t.contextType;return"object"==typeof o&&null!==o?o=io(o):(r=xr(t)?br:gr.current,o=(s=null!=(s=t.contextTypes))?yr(e,r):hr),t=new t(n,o),e.memoizedState=null!==t.state&&void 0!==t.state?t.state:null,t.updater=vo,e.stateNode=t,t._reactInternalFiber=e,s&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=r,e.__reactInternalMemoizedMaskedChildContext=o),t}function Eo(e,t,n,s){e=t.state,"function"==typeof t.componentWillReceiveProps&&t.componentWillReceiveProps(n,s),"function"==typeof t.UNSAFE_componentWillReceiveProps&&t.UNSAFE_componentWillReceiveProps(n,s),t.state!==e&&vo.enqueueReplaceState(t,t.state,null)}function Lo(e,t,n,s){var r=e.stateNode;r.props=n,r.state=e.memoizedState,r.refs=yo,lo(e);var o=t.contextType;"object"==typeof o&&null!==o?r.context=io(o):(o=xr(t)?br:gr.current,r.context=yr(e,o)),go(e,n,r,s),r.state=e.memoizedState,"function"==typeof(o=t.getDerivedStateFromProps)&&(xo(e,t,o,n),r.state=e.memoizedState),"function"==typeof t.getDerivedStateFromProps||"function"==typeof r.getSnapshotBeforeUpdate||"function"!=typeof r.UNSAFE_componentWillMount&&"function"!=typeof r.componentWillMount||(t=r.state,"function"==typeof r.componentWillMount&&r.componentWillMount(),"function"==typeof r.UNSAFE_componentWillMount&&r.UNSAFE_componentWillMount(),t!==r.state&&vo.enqueueReplaceState(r,r.state,null),go(e,n,r,s),r.state=e.memoizedState),"function"==typeof r.componentDidMount&&(e.effectTag|=4)}var Mo=Array.isArray;function ko(e,t,n){if(null!==(e=n.ref)&&"function"!=typeof e&&"object"!=typeof e){if(n._owner){if(n=n._owner){if(1!==n.tag)throw Error(a(309));var s=n.stateNode}if(!s)throw Error(a(147,e));var r=""+e;return null!==t&&null!==t.ref&&"function"==typeof t.ref&&t.ref._stringRef===r?t.ref:(t=function(e){var t=s.refs;t===yo&&(t=s.refs={}),null===e?delete t[r]:t[r]=e},t._stringRef=r,t)}if("string"!=typeof e)throw Error(a(284));if(!n._owner)throw Error(a(290,e))}return e}function _o(e,t){if("textarea"!==e.type)throw Error(a(31,"[object Object]"===Object.prototype.toString.call(t)?"object with keys {"+Object.keys(t).join(", ")+"}":t,""))}function wo(e){function t(t,n){if(e){var s=t.lastEffect;null!==s?(s.nextEffect=n,t.lastEffect=n):t.firstEffect=t.lastEffect=n,n.nextEffect=null,n.effectTag=8}}function n(n,s){if(!e)return null;for(;null!==s;)t(n,s),s=s.sibling;return null}function s(e,t){for(e=new Map;null!==t;)null!==t.key?e.set(t.key,t):e.set(t.index,t),t=t.sibling;return e}function r(e,t){return(e=Tc(e,t)).index=0,e.sibling=null,e}function o(t,n,s){return t.index=s,e?null!==(s=t.alternate)?(s=s.index)<n?(t.effectTag=2,n):s:(t.effectTag=2,n):n}function i(t){return e&&null===t.alternate&&(t.effectTag=2),t}function c(e,t,n,s){return null===t||6!==t.tag?((t=Fc(n,e.mode,s)).return=e,t):((t=r(t,n)).return=e,t)}function l(e,t,n,s){return null!==t&&t.elementType===n.type?((s=r(t,n.props)).ref=ko(e,t,n),s.return=e,s):((s=Oc(n.type,n.key,n.props,null,e.mode,s)).ref=ko(e,t,n),s.return=e,s)}function d(e,t,n,s){return null===t||4!==t.tag||t.stateNode.containerInfo!==n.containerInfo||t.stateNode.implementation!==n.implementation?((t=Nc(n,e.mode,s)).return=e,t):((t=r(t,n.children||[])).return=e,t)}function p(e,t,n,s,o){return null===t||7!==t.tag?((t=Pc(n,e.mode,s,o)).return=e,t):((t=r(t,n)).return=e,t)}function u(e,t,n){if("string"==typeof t||"number"==typeof t)return(t=Fc(""+t,e.mode,n)).return=e,t;if("object"==typeof t&&null!==t){switch(t.$$typeof){case ee:return(n=Oc(t.type,t.key,t.props,null,e.mode,n)).ref=ko(e,null,t),n.return=e,n;case te:return(t=Nc(t,e.mode,n)).return=e,t}if(Mo(t)||ge(t))return(t=Pc(t,e.mode,n,null)).return=e,t;_o(e,t)}return null}function m(e,t,n,s){var r=null!==t?t.key:null;if("string"==typeof n||"number"==typeof n)return null!==r?null:c(e,t,""+n,s);if("object"==typeof n&&null!==n){switch(n.$$typeof){case ee:return n.key===r?n.type===ne?p(e,t,n.props.children,s,r):l(e,t,n,s):null;case te:return n.key===r?d(e,t,n,s):null}if(Mo(n)||ge(n))return null!==r?null:p(e,t,n,s,null);_o(e,n)}return null}function h(e,t,n,s,r){if("string"==typeof s||"number"==typeof s)return c(t,e=e.get(n)||null,""+s,r);if("object"==typeof s&&null!==s){switch(s.$$typeof){case ee:return e=e.get(null===s.key?n:s.key)||null,s.type===ne?p(t,e,s.props.children,r,s.key):l(t,e,s,r);case te:return d(t,e=e.get(null===s.key?n:s.key)||null,s,r)}if(Mo(s)||ge(s))return p(t,e=e.get(n)||null,s,r,null);_o(t,s)}return null}function g(r,a,i,c){for(var l=null,d=null,p=a,g=a=0,f=null;null!==p&&g<i.length;g++){p.index>g?(f=p,p=null):f=p.sibling;var b=m(r,p,i[g],c);if(null===b){null===p&&(p=f);break}e&&p&&null===b.alternate&&t(r,p),a=o(b,a,g),null===d?l=b:d.sibling=b,d=b,p=f}if(g===i.length)return n(r,p),l;if(null===p){for(;g<i.length;g++)null!==(p=u(r,i[g],c))&&(a=o(p,a,g),null===d?l=p:d.sibling=p,d=p);return l}for(p=s(r,p);g<i.length;g++)null!==(f=h(p,r,g,i[g],c))&&(e&&null!==f.alternate&&p.delete(null===f.key?g:f.key),a=o(f,a,g),null===d?l=f:d.sibling=f,d=f);return e&&p.forEach((function(e){return t(r,e)})),l}function f(r,i,c,l){var d=ge(c);if("function"!=typeof d)throw Error(a(150));if(null==(c=d.call(c)))throw Error(a(151));for(var p=d=null,g=i,f=i=0,b=null,y=c.next();null!==g&&!y.done;f++,y=c.next()){g.index>f?(b=g,g=null):b=g.sibling;var x=m(r,g,y.value,l);if(null===x){null===g&&(g=b);break}e&&g&&null===x.alternate&&t(r,g),i=o(x,i,f),null===p?d=x:p.sibling=x,p=x,g=b}if(y.done)return n(r,g),d;if(null===g){for(;!y.done;f++,y=c.next())null!==(y=u(r,y.value,l))&&(i=o(y,i,f),null===p?d=y:p.sibling=y,p=y);return d}for(g=s(r,g);!y.done;f++,y=c.next())null!==(y=h(g,r,f,y.value,l))&&(e&&null!==y.alternate&&g.delete(null===y.key?f:y.key),i=o(y,i,f),null===p?d=y:p.sibling=y,p=y);return e&&g.forEach((function(e){return t(r,e)})),d}return function(e,s,o,c){var l="object"==typeof o&&null!==o&&o.type===ne&&null===o.key;l&&(o=o.props.children);var d="object"==typeof o&&null!==o;if(d)switch(o.$$typeof){case ee:e:{for(d=o.key,l=s;null!==l;){if(l.key===d){if(7===l.tag){if(o.type===ne){n(e,l.sibling),(s=r(l,o.props.children)).return=e,e=s;break e}}else if(l.elementType===o.type){n(e,l.sibling),(s=r(l,o.props)).ref=ko(e,l,o),s.return=e,e=s;break e}n(e,l);break}t(e,l),l=l.sibling}o.type===ne?((s=Pc(o.props.children,e.mode,c,o.key)).return=e,e=s):((c=Oc(o.type,o.key,o.props,null,e.mode,c)).ref=ko(e,s,o),c.return=e,e=c)}return i(e);case te:e:{for(l=o.key;null!==s;){if(s.key===l){if(4===s.tag&&s.stateNode.containerInfo===o.containerInfo&&s.stateNode.implementation===o.implementation){n(e,s.sibling),(s=r(s,o.children||[])).return=e,e=s;break e}n(e,s);break}t(e,s),s=s.sibling}(s=Nc(o,e.mode,c)).return=e,e=s}return i(e)}if("string"==typeof o||"number"==typeof o)return o=""+o,null!==s&&6===s.tag?(n(e,s.sibling),(s=r(s,o)).return=e,e=s):(n(e,s),(s=Fc(o,e.mode,c)).return=e,e=s),i(e);if(Mo(o))return g(e,s,o,c);if(ge(o))return f(e,s,o,c);if(d&&_o(e,o),void 0===o&&!l)switch(e.tag){case 1:case 0:throw e=e.type,Error(a(152,e.displayName||e.name||"Component"))}return n(e,s)}}var Io=wo(!0),Co=wo(!1),Qo={},Ro={current:Qo},Do={current:Qo},To={current:Qo};function Oo(e){if(e===Qo)throw Error(a(174));return e}function Po(e,t){switch(mr(To,t),mr(Do,e),mr(Ro,Qo),e=t.nodeType){case 9:case 11:t=(t=t.documentElement)?t.namespaceURI:Fe(null,"");break;default:t=Fe(t=(e=8===e?t.parentNode:t).namespaceURI||null,e=e.tagName)}ur(Ro),mr(Ro,t)}function Fo(){ur(Ro),ur(Do),ur(To)}function No(e){Oo(To.current);var t=Oo(Ro.current),n=Fe(t,e.type);t!==n&&(mr(Do,e),mr(Ro,n))}function Bo(e){Do.current===e&&(ur(Ro),ur(Do))}var Vo={current:0};function Go(e){for(var t=e;null!==t;){if(13===t.tag){var n=t.memoizedState;if(null!==n&&(null===(n=n.dehydrated)||n.data===bn||n.data===yn))return t}else if(19===t.tag&&void 0!==t.memoizedProps.revealOrder){if(0!=(64&t.effectTag))return t}else if(null!==t.child){t.child.return=t,t=t.child;continue}if(t===e)break;for(;null===t.sibling;){if(null===t.return||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}return null}function Wo(e,t){return{responder:e,props:t}}var Uo=X.ReactCurrentDispatcher,zo=X.ReactCurrentBatchConfig,Ho=0,jo=null,$o=null,Zo=null,Ko=!1;function Xo(){throw Error(a(321))}function Yo(e,t){if(null===t)return!1;for(var n=0;n<t.length&&n<e.length;n++)if(!Gs(e[n],t[n]))return!1;return!0}function qo(e,t,n,s,r,o){if(Ho=o,jo=t,t.memoizedState=null,t.updateQueue=null,t.expirationTime=0,Uo.current=null===e||null===e.memoizedState?Aa:Sa,e=n(s,r),t.expirationTime===Ho){o=0;do{if(t.expirationTime=0,!(25>o))throw Error(a(301));o+=1,Zo=$o=null,t.updateQueue=null,Uo.current=Ea,e=n(s,r)}while(t.expirationTime===Ho)}if(Uo.current=va,t=null!==$o&&null!==$o.next,Ho=0,Zo=$o=jo=null,Ko=!1,t)throw Error(a(300));return e}function Jo(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return null===Zo?jo.memoizedState=Zo=e:Zo=Zo.next=e,Zo}function ea(){if(null===$o){var e=jo.alternate;e=null!==e?e.memoizedState:null}else e=$o.next;var t=null===Zo?jo.memoizedState:Zo.next;if(null!==t)Zo=t,$o=e;else{if(null===e)throw Error(a(310));e={memoizedState:($o=e).memoizedState,baseState:$o.baseState,baseQueue:$o.baseQueue,queue:$o.queue,next:null},null===Zo?jo.memoizedState=Zo=e:Zo=Zo.next=e}return Zo}function ta(e,t){return"function"==typeof t?t(e):t}function na(e){var t=ea(),n=t.queue;if(null===n)throw Error(a(311));n.lastRenderedReducer=e;var s=$o,r=s.baseQueue,o=n.pending;if(null!==o){if(null!==r){var i=r.next;r.next=o.next,o.next=i}s.baseQueue=r=o,n.pending=null}if(null!==r){r=r.next,s=s.baseState;var c=i=o=null,l=r;do{var d=l.expirationTime;if(d<Ho){var p={expirationTime:l.expirationTime,suspenseConfig:l.suspenseConfig,action:l.action,eagerReducer:l.eagerReducer,eagerState:l.eagerState,next:null};null===c?(i=c=p,o=s):c=c.next=p,d>jo.expirationTime&&(jo.expirationTime=d,hc(d))}else null!==c&&(c=c.next={expirationTime:1073741823,suspenseConfig:l.suspenseConfig,action:l.action,eagerReducer:l.eagerReducer,eagerState:l.eagerState,next:null}),mc(d,l.suspenseConfig),s=l.eagerReducer===e?l.eagerState:e(s,l.action);l=l.next}while(null!==l&&l!==r);null===c?o=s:c.next=i,Gs(s,t.memoizedState)||(Ta=!0),t.memoizedState=s,t.baseState=o,t.baseQueue=c,n.lastRenderedState=s}return[t.memoizedState,n.dispatch]}function sa(e){var t=ea(),n=t.queue;if(null===n)throw Error(a(311));n.lastRenderedReducer=e;var s=n.dispatch,r=n.pending,o=t.memoizedState;if(null!==r){n.pending=null;var i=r=r.next;do{o=e(o,i.action),i=i.next}while(i!==r);Gs(o,t.memoizedState)||(Ta=!0),t.memoizedState=o,null===t.baseQueue&&(t.baseState=o),n.lastRenderedState=o}return[o,s]}function ra(e){var t=Jo();return"function"==typeof e&&(e=e()),t.memoizedState=t.baseState=e,e=(e=t.queue={pending:null,dispatch:null,lastRenderedReducer:ta,lastRenderedState:e}).dispatch=xa.bind(null,jo,e),[t.memoizedState,e]}function oa(e,t,n,s){return e={tag:e,create:t,destroy:n,deps:s,next:null},null===(t=jo.updateQueue)?(t={lastEffect:null},jo.updateQueue=t,t.lastEffect=e.next=e):null===(n=t.lastEffect)?t.lastEffect=e.next=e:(s=n.next,n.next=e,e.next=s,t.lastEffect=e),e}function aa(){return ea().memoizedState}function ia(e,t,n,s){var r=Jo();jo.effectTag|=e,r.memoizedState=oa(1|t,n,void 0,void 0===s?null:s)}function ca(e,t,n,s){var r=ea();s=void 0===s?null:s;var o=void 0;if(null!==$o){var a=$o.memoizedState;if(o=a.destroy,null!==s&&Yo(s,a.deps))return void oa(t,n,o,s)}jo.effectTag|=e,r.memoizedState=oa(1|t,n,o,s)}function la(e,t){return ia(516,4,e,t)}function da(e,t){return ca(516,4,e,t)}function pa(e,t){return ca(4,2,e,t)}function ua(e,t){return"function"==typeof t?(e=e(),t(e),function(){t(null)}):null!=t?(e=e(),t.current=e,function(){t.current=null}):void 0}function ma(e,t,n){return n=null!=n?n.concat([e]):null,ca(4,2,ua.bind(null,t,e),n)}function ha(){}function ga(e,t){return Jo().memoizedState=[e,void 0===t?null:t],e}function fa(e,t){var n=ea();t=void 0===t?null:t;var s=n.memoizedState;return null!==s&&null!==t&&Yo(t,s[1])?s[0]:(n.memoizedState=[e,t],e)}function ba(e,t){var n=ea();t=void 0===t?null:t;var s=n.memoizedState;return null!==s&&null!==t&&Yo(t,s[1])?s[0]:(e=e(),n.memoizedState=[e,t],e)}function ya(e,t,n){var s=zr();jr(98>s?98:s,(function(){e(!0)})),jr(97<s?97:s,(function(){var s=zo.suspense;zo.suspense=void 0===t?null:t;try{e(!1),n()}finally{zo.suspense=s}}))}function xa(e,t,n){var s=ec(),r=bo.suspense;r={expirationTime:s=tc(s,e,r),suspenseConfig:r,action:n,eagerReducer:null,eagerState:null,next:null};var o=t.pending;if(null===o?r.next=r:(r.next=o.next,o.next=r),t.pending=r,o=e.alternate,e===jo||null!==o&&o===jo)Ko=!0,r.expirationTime=Ho,jo.expirationTime=Ho;else{if(0===e.expirationTime&&(null===o||0===o.expirationTime)&&null!==(o=t.lastRenderedReducer))try{var a=t.lastRenderedState,i=o(a,n);if(r.eagerReducer=o,r.eagerState=i,Gs(i,a))return}catch(c){}nc(e,s)}}var va={readContext:io,useCallback:Xo,useContext:Xo,useEffect:Xo,useImperativeHandle:Xo,useLayoutEffect:Xo,useMemo:Xo,useReducer:Xo,useRef:Xo,useState:Xo,useDebugValue:Xo,useResponder:Xo,useDeferredValue:Xo,useTransition:Xo},Aa={readContext:io,useCallback:ga,useContext:io,useEffect:la,useImperativeHandle:function(e,t,n){return n=null!=n?n.concat([e]):null,ia(4,2,ua.bind(null,t,e),n)},useLayoutEffect:function(e,t){return ia(4,2,e,t)},useMemo:function(e,t){var n=Jo();return t=void 0===t?null:t,e=e(),n.memoizedState=[e,t],e},useReducer:function(e,t,n){var s=Jo();return t=void 0!==n?n(t):t,s.memoizedState=s.baseState=t,e=(e=s.queue={pending:null,dispatch:null,lastRenderedReducer:e,lastRenderedState:t}).dispatch=xa.bind(null,jo,e),[s.memoizedState,e]},useRef:function(e){return e={current:e},Jo().memoizedState=e},useState:ra,useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=ra(e),s=n[0],r=n[1];return la((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=ra(!1),n=t[0];return t=t[1],[ga(ya.bind(null,t,e),[t,e]),n]}},Sa={readContext:io,useCallback:fa,useContext:io,useEffect:da,useImperativeHandle:ma,useLayoutEffect:pa,useMemo:ba,useReducer:na,useRef:aa,useState:function(){return na(ta)},useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=na(ta),s=n[0],r=n[1];return da((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=na(ta),n=t[0];return t=t[1],[fa(ya.bind(null,t,e),[t,e]),n]}},Ea={readContext:io,useCallback:fa,useContext:io,useEffect:da,useImperativeHandle:ma,useLayoutEffect:pa,useMemo:ba,useReducer:sa,useRef:aa,useState:function(){return sa(ta)},useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=sa(ta),s=n[0],r=n[1];return da((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=sa(ta),n=t[0];return t=t[1],[fa(ya.bind(null,t,e),[t,e]),n]}},La=null,Ma=null,ka=!1;function _a(e,t){var n=Rc(5,null,null,0);n.elementType="DELETED",n.type="DELETED",n.stateNode=t,n.return=e,n.effectTag=8,null!==e.lastEffect?(e.lastEffect.nextEffect=n,e.lastEffect=n):e.firstEffect=e.lastEffect=n}function wa(e,t){switch(e.tag){case 5:var n=e.type;return null!==(t=1!==t.nodeType||n.toLowerCase()!==t.nodeName.toLowerCase()?null:t)&&(e.stateNode=t,!0);case 6:return null!==(t=""===e.pendingProps||3!==t.nodeType?null:t)&&(e.stateNode=t,!0);default:return!1}}function Ia(e){if(ka){var t=Ma;if(t){var n=t;if(!wa(e,t)){if(!(t=Mn(n.nextSibling))||!wa(e,t))return e.effectTag=-1025&e.effectTag|2,ka=!1,void(La=e);_a(La,n)}La=e,Ma=Mn(t.firstChild)}else e.effectTag=-1025&e.effectTag|2,ka=!1,La=e}}function Ca(e){for(e=e.return;null!==e&&5!==e.tag&&3!==e.tag&&13!==e.tag;)e=e.return;La=e}function Qa(e){if(e!==La)return!1;if(!ka)return Ca(e),ka=!0,!1;var t=e.type;if(5!==e.tag||"head"!==t&&"body"!==t&&!Sn(t,e.memoizedProps))for(t=Ma;t;)_a(e,t),t=Mn(t.nextSibling);if(Ca(e),13===e.tag){if(!(e=null!==(e=e.memoizedState)?e.dehydrated:null))throw Error(a(317));e:{for(e=e.nextSibling,t=0;e;){if(8===e.nodeType){var n=e.data;if("/$"===n){if(0===t){Ma=Mn(e.nextSibling);break e}t--}else"$"!==n&&n!==yn&&n!==bn||t++}e=e.nextSibling}Ma=null}}else Ma=La?Mn(e.stateNode.nextSibling):null;return!0}function Ra(){Ma=La=null,ka=!1}var Da=X.ReactCurrentOwner,Ta=!1;function Oa(e,t,n,s){t.child=null===e?Co(t,null,n,s):Io(t,e.child,n,s)}function Pa(e,t,n,s,r){n=n.render;var o=t.ref;return ao(t,r),s=qo(e,t,n,s,o,r),null===e||Ta?(t.effectTag|=1,Oa(e,t,s,r),t.child):(t.updateQueue=e.updateQueue,t.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),qa(e,t,r))}function Fa(e,t,n,s,r,o){if(null===e){var a=n.type;return"function"!=typeof a||Dc(a)||void 0!==a.defaultProps||null!==n.compare||void 0!==n.defaultProps?((e=Oc(n.type,null,s,null,t.mode,o)).ref=t.ref,e.return=t,t.child=e):(t.tag=15,t.type=a,Na(e,t,a,s,r,o))}return a=e.child,r<o&&(r=a.memoizedProps,(n=null!==(n=n.compare)?n:Us)(r,s)&&e.ref===t.ref)?qa(e,t,o):(t.effectTag|=1,(e=Tc(a,s)).ref=t.ref,e.return=t,t.child=e)}function Na(e,t,n,s,r,o){return null!==e&&Us(e.memoizedProps,s)&&e.ref===t.ref&&(Ta=!1,r<o)?(t.expirationTime=e.expirationTime,qa(e,t,o)):Va(e,t,n,s,o)}function Ba(e,t){var n=t.ref;(null===e&&null!==n||null!==e&&e.ref!==n)&&(t.effectTag|=128)}function Va(e,t,n,s,r){var o=xr(n)?br:gr.current;return o=yr(t,o),ao(t,r),n=qo(e,t,n,s,o,r),null===e||Ta?(t.effectTag|=1,Oa(e,t,n,r),t.child):(t.updateQueue=e.updateQueue,t.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),qa(e,t,r))}function Ga(e,t,n,s,r){if(xr(n)){var o=!0;Er(t)}else o=!1;if(ao(t,r),null===t.stateNode)null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),So(t,n,s),Lo(t,n,s,r),s=!0;else if(null===e){var a=t.stateNode,i=t.memoizedProps;a.props=i;var c=a.context,l=n.contextType;"object"==typeof l&&null!==l?l=io(l):l=yr(t,l=xr(n)?br:gr.current);var d=n.getDerivedStateFromProps,p="function"==typeof d||"function"==typeof a.getSnapshotBeforeUpdate;p||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(i!==s||c!==l)&&Eo(t,a,s,l),co=!1;var u=t.memoizedState;a.state=u,go(t,s,a,r),c=t.memoizedState,i!==s||u!==c||fr.current||co?("function"==typeof d&&(xo(t,n,d,s),c=t.memoizedState),(i=co||Ao(t,n,i,s,u,c,l))?(p||"function"!=typeof a.UNSAFE_componentWillMount&&"function"!=typeof a.componentWillMount||("function"==typeof a.componentWillMount&&a.componentWillMount(),"function"==typeof a.UNSAFE_componentWillMount&&a.UNSAFE_componentWillMount()),"function"==typeof a.componentDidMount&&(t.effectTag|=4)):("function"==typeof a.componentDidMount&&(t.effectTag|=4),t.memoizedProps=s,t.memoizedState=c),a.props=s,a.state=c,a.context=l,s=i):("function"==typeof a.componentDidMount&&(t.effectTag|=4),s=!1)}else a=t.stateNode,po(e,t),i=t.memoizedProps,a.props=t.type===t.elementType?i:qr(t.type,i),c=a.context,"object"==typeof(l=n.contextType)&&null!==l?l=io(l):l=yr(t,l=xr(n)?br:gr.current),(p="function"==typeof(d=n.getDerivedStateFromProps)||"function"==typeof a.getSnapshotBeforeUpdate)||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(i!==s||c!==l)&&Eo(t,a,s,l),co=!1,c=t.memoizedState,a.state=c,go(t,s,a,r),u=t.memoizedState,i!==s||c!==u||fr.current||co?("function"==typeof d&&(xo(t,n,d,s),u=t.memoizedState),(d=co||Ao(t,n,i,s,c,u,l))?(p||"function"!=typeof a.UNSAFE_componentWillUpdate&&"function"!=typeof a.componentWillUpdate||("function"==typeof a.componentWillUpdate&&a.componentWillUpdate(s,u,l),"function"==typeof a.UNSAFE_componentWillUpdate&&a.UNSAFE_componentWillUpdate(s,u,l)),"function"==typeof a.componentDidUpdate&&(t.effectTag|=4),"function"==typeof a.getSnapshotBeforeUpdate&&(t.effectTag|=256)):("function"!=typeof a.componentDidUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=256),t.memoizedProps=s,t.memoizedState=u),a.props=s,a.state=u,a.context=l,s=d):("function"!=typeof a.componentDidUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=256),s=!1);return Wa(e,t,n,s,o,r)}function Wa(e,t,n,s,r,o){Ba(e,t);var a=0!=(64&t.effectTag);if(!s&&!a)return r&&Lr(t,n,!1),qa(e,t,o);s=t.stateNode,Da.current=t;var i=a&&"function"!=typeof n.getDerivedStateFromError?null:s.render();return t.effectTag|=1,null!==e&&a?(t.child=Io(t,e.child,null,o),t.child=Io(t,null,i,o)):Oa(e,t,i,o),t.memoizedState=s.state,r&&Lr(t,n,!0),t.child}function Ua(e){var t=e.stateNode;t.pendingContext?Ar(0,t.pendingContext,t.pendingContext!==t.context):t.context&&Ar(0,t.context,!1),Po(e,t.containerInfo)}var za,Ha,ja,$a={dehydrated:null,retryTime:0};function Za(e,t,n){var s,r=t.mode,o=t.pendingProps,a=Vo.current,i=!1;if((s=0!=(64&t.effectTag))||(s=0!=(2&a)&&(null===e||null!==e.memoizedState)),s?(i=!0,t.effectTag&=-65):null!==e&&null===e.memoizedState||void 0===o.fallback||!0===o.unstable_avoidThisFallback||(a|=1),mr(Vo,1&a),null===e){if(void 0!==o.fallback&&Ia(t),i){if(i=o.fallback,(o=Pc(null,r,0,null)).return=t,0==(2&t.mode))for(e=null!==t.memoizedState?t.child.child:t.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(n=Pc(i,r,n,null)).return=t,o.sibling=n,t.memoizedState=$a,t.child=o,n}return r=o.children,t.memoizedState=null,t.child=Co(t,null,r,n)}if(null!==e.memoizedState){if(r=(e=e.child).sibling,i){if(o=o.fallback,(n=Tc(e,e.pendingProps)).return=t,0==(2&t.mode)&&(i=null!==t.memoizedState?t.child.child:t.child)!==e.child)for(n.child=i;null!==i;)i.return=n,i=i.sibling;return(r=Tc(r,o)).return=t,n.sibling=r,n.childExpirationTime=0,t.memoizedState=$a,t.child=n,r}return n=Io(t,e.child,o.children,n),t.memoizedState=null,t.child=n}if(e=e.child,i){if(i=o.fallback,(o=Pc(null,r,0,null)).return=t,o.child=e,null!==e&&(e.return=o),0==(2&t.mode))for(e=null!==t.memoizedState?t.child.child:t.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(n=Pc(i,r,n,null)).return=t,o.sibling=n,n.effectTag|=2,o.childExpirationTime=0,t.memoizedState=$a,t.child=o,n}return t.memoizedState=null,t.child=Io(t,e,o.children,n)}function Ka(e,t){e.expirationTime<t&&(e.expirationTime=t);var n=e.alternate;null!==n&&n.expirationTime<t&&(n.expirationTime=t),oo(e.return,t)}function Xa(e,t,n,s,r,o){var a=e.memoizedState;null===a?e.memoizedState={isBackwards:t,rendering:null,renderingStartTime:0,last:s,tail:n,tailExpiration:0,tailMode:r,lastEffect:o}:(a.isBackwards=t,a.rendering=null,a.renderingStartTime=0,a.last=s,a.tail=n,a.tailExpiration=0,a.tailMode=r,a.lastEffect=o)}function Ya(e,t,n){var s=t.pendingProps,r=s.revealOrder,o=s.tail;if(Oa(e,t,s.children,n),0!=(2&(s=Vo.current)))s=1&s|2,t.effectTag|=64;else{if(null!==e&&0!=(64&e.effectTag))e:for(e=t.child;null!==e;){if(13===e.tag)null!==e.memoizedState&&Ka(e,n);else if(19===e.tag)Ka(e,n);else if(null!==e.child){e.child.return=e,e=e.child;continue}if(e===t)break e;for(;null===e.sibling;){if(null===e.return||e.return===t)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}s&=1}if(mr(Vo,s),0==(2&t.mode))t.memoizedState=null;else switch(r){case"forwards":for(n=t.child,r=null;null!==n;)null!==(e=n.alternate)&&null===Go(e)&&(r=n),n=n.sibling;null===(n=r)?(r=t.child,t.child=null):(r=n.sibling,n.sibling=null),Xa(t,!1,r,n,o,t.lastEffect);break;case"backwards":for(n=null,r=t.child,t.child=null;null!==r;){if(null!==(e=r.alternate)&&null===Go(e)){t.child=r;break}e=r.sibling,r.sibling=n,n=r,r=e}Xa(t,!0,n,null,o,t.lastEffect);break;case"together":Xa(t,!1,null,null,void 0,t.lastEffect);break;default:t.memoizedState=null}return t.child}function qa(e,t,n){null!==e&&(t.dependencies=e.dependencies);var s=t.expirationTime;if(0!==s&&hc(s),t.childExpirationTime<n)return null;if(null!==e&&t.child!==e.child)throw Error(a(153));if(null!==t.child){for(n=Tc(e=t.child,e.pendingProps),t.child=n,n.return=t;null!==e.sibling;)e=e.sibling,(n=n.sibling=Tc(e,e.pendingProps)).return=t;n.sibling=null}return t.child}function Ja(e,t){switch(e.tailMode){case"hidden":t=e.tail;for(var n=null;null!==t;)null!==t.alternate&&(n=t),t=t.sibling;null===n?e.tail=null:n.sibling=null;break;case"collapsed":n=e.tail;for(var s=null;null!==n;)null!==n.alternate&&(s=n),n=n.sibling;null===s?t||null===e.tail?e.tail=null:e.tail.sibling=null:s.sibling=null}}function ei(e,t,n){var s=t.pendingProps;switch(t.tag){case 2:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return null;case 1:case 17:return xr(t.type)&&vr(),null;case 3:return Fo(),ur(fr),ur(gr),(n=t.stateNode).pendingContext&&(n.context=n.pendingContext,n.pendingContext=null),null!==e&&null!==e.child||!Qa(t)||(t.effectTag|=4),null;case 5:Bo(t),n=Oo(To.current);var o=t.type;if(null!==e&&null!=t.stateNode)Ha(e,t,o,s,n),e.ref!==t.ref&&(t.effectTag|=128);else{if(!s){if(null===t.stateNode)throw Error(a(166));return null}if(e=Oo(Ro.current),Qa(t)){s=t.stateNode,o=t.type;var i=t.memoizedProps;switch(s[wn]=t,s[In]=i,o){case"iframe":case"object":case"embed":Zt("load",s);break;case"video":case"audio":for(e=0;e<Ye.length;e++)Zt(Ye[e],s);break;case"source":Zt("error",s);break;case"img":case"image":case"link":Zt("error",s),Zt("load",s);break;case"form":Zt("reset",s),Zt("submit",s);break;case"details":Zt("toggle",s);break;case"input":Ee(s,i),Zt("invalid",s),ln(n,"onChange");break;case"select":s._wrapperState={wasMultiple:!!i.multiple},Zt("invalid",s),ln(n,"onChange");break;case"textarea":Qe(s,i),Zt("invalid",s),ln(n,"onChange")}for(var c in on(o,i),e=null,i)if(i.hasOwnProperty(c)){var l=i[c];"children"===c?"string"==typeof l?s.textContent!==l&&(e=["children",l]):"number"==typeof l&&s.textContent!==""+l&&(e=["children",""+l]):L.hasOwnProperty(c)&&null!=l&&ln(n,c)}switch(o){case"input":ve(s),ke(s,i,!0);break;case"textarea":ve(s),De(s);break;case"select":case"option":break;default:"function"==typeof i.onClick&&(s.onclick=dn)}n=e,t.updateQueue=n,null!==n&&(t.effectTag|=4)}else{switch(c=9===n.nodeType?n:n.ownerDocument,e===cn&&(e=Pe(o)),e===cn?"script"===o?((e=c.createElement("div")).innerHTML="<script><\/script>",e=e.removeChild(e.firstChild)):"string"==typeof s.is?e=c.createElement(o,{is:s.is}):(e=c.createElement(o),"select"===o&&(c=e,s.multiple?c.multiple=!0:s.size&&(c.size=s.size))):e=c.createElementNS(e,o),e[wn]=t,e[In]=s,za(e,t),t.stateNode=e,c=an(o,s),o){case"iframe":case"object":case"embed":Zt("load",e),l=s;break;case"video":case"audio":for(l=0;l<Ye.length;l++)Zt(Ye[l],e);l=s;break;case"source":Zt("error",e),l=s;break;case"img":case"image":case"link":Zt("error",e),Zt("load",e),l=s;break;case"form":Zt("reset",e),Zt("submit",e),l=s;break;case"details":Zt("toggle",e),l=s;break;case"input":Ee(e,s),l=Se(e,s),Zt("invalid",e),ln(n,"onChange");break;case"option":l=we(e,s);break;case"select":e._wrapperState={wasMultiple:!!s.multiple},l=r({},s,{value:void 0}),Zt("invalid",e),ln(n,"onChange");break;case"textarea":Qe(e,s),l=Ce(e,s),Zt("invalid",e),ln(n,"onChange");break;default:l=s}on(o,l);var d=l;for(i in d)if(d.hasOwnProperty(i)){var p=d[i];"style"===i?sn(e,p):"dangerouslySetInnerHTML"===i?null!=(p=p?p.__html:void 0)&&Ve(e,p):"children"===i?"string"==typeof p?("textarea"!==o||""!==p)&&Ge(e,p):"number"==typeof p&&Ge(e,""+p):"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&"autoFocus"!==i&&(L.hasOwnProperty(i)?null!=p&&ln(n,i):null!=p&&Y(e,i,p,c))}switch(o){case"input":ve(e),ke(e,s,!1);break;case"textarea":ve(e),De(e);break;case"option":null!=s.value&&e.setAttribute("value",""+ye(s.value));break;case"select":e.multiple=!!s.multiple,null!=(n=s.value)?Ie(e,!!s.multiple,n,!1):null!=s.defaultValue&&Ie(e,!!s.multiple,s.defaultValue,!0);break;default:"function"==typeof l.onClick&&(e.onclick=dn)}An(o,s)&&(t.effectTag|=4)}null!==t.ref&&(t.effectTag|=128)}return null;case 6:if(e&&null!=t.stateNode)ja(0,t,e.memoizedProps,s);else{if("string"!=typeof s&&null===t.stateNode)throw Error(a(166));n=Oo(To.current),Oo(Ro.current),Qa(t)?(n=t.stateNode,s=t.memoizedProps,n[wn]=t,n.nodeValue!==s&&(t.effectTag|=4)):((n=(9===n.nodeType?n:n.ownerDocument).createTextNode(s))[wn]=t,t.stateNode=n)}return null;case 13:return ur(Vo),s=t.memoizedState,0!=(64&t.effectTag)?(t.expirationTime=n,t):(n=null!==s,s=!1,null===e?void 0!==t.memoizedProps.fallback&&Qa(t):(s=null!==(o=e.memoizedState),n||null===o||null!==(o=e.child.sibling)&&(null!==(i=t.firstEffect)?(t.firstEffect=o,o.nextEffect=i):(t.firstEffect=t.lastEffect=o,o.nextEffect=null),o.effectTag=8)),n&&!s&&0!=(2&t.mode)&&(null===e&&!0!==t.memoizedProps.unstable_avoidThisFallback||0!=(1&Vo.current)?Oi===wi&&(Oi=Ii):(Oi!==wi&&Oi!==Ii||(Oi=Ci),0!==Vi&&null!==Ri&&(Gc(Ri,Ti),Wc(Ri,Vi)))),(n||s)&&(t.effectTag|=4),null);case 4:return Fo(),null;case 10:return ro(t),null;case 19:if(ur(Vo),null===(s=t.memoizedState))return null;if(o=0!=(64&t.effectTag),null===(i=s.rendering)){if(o)Ja(s,!1);else if(Oi!==wi||null!==e&&0!=(64&e.effectTag))for(i=t.child;null!==i;){if(null!==(e=Go(i))){for(t.effectTag|=64,Ja(s,!1),null!==(o=e.updateQueue)&&(t.updateQueue=o,t.effectTag|=4),null===s.lastEffect&&(t.firstEffect=null),t.lastEffect=s.lastEffect,s=t.child;null!==s;)i=n,(o=s).effectTag&=2,o.nextEffect=null,o.firstEffect=null,o.lastEffect=null,null===(e=o.alternate)?(o.childExpirationTime=0,o.expirationTime=i,o.child=null,o.memoizedProps=null,o.memoizedState=null,o.updateQueue=null,o.dependencies=null):(o.childExpirationTime=e.childExpirationTime,o.expirationTime=e.expirationTime,o.child=e.child,o.memoizedProps=e.memoizedProps,o.memoizedState=e.memoizedState,o.updateQueue=e.updateQueue,i=e.dependencies,o.dependencies=null===i?null:{expirationTime:i.expirationTime,firstContext:i.firstContext,responders:i.responders}),s=s.sibling;return mr(Vo,1&Vo.current|2),t.child}i=i.sibling}}else{if(!o)if(null!==(e=Go(i))){if(t.effectTag|=64,o=!0,null!==(n=e.updateQueue)&&(t.updateQueue=n,t.effectTag|=4),Ja(s,!0),null===s.tail&&"hidden"===s.tailMode&&!i.alternate)return null!==(t=t.lastEffect=s.lastEffect)&&(t.nextEffect=null),null}else 2*Ur()-s.renderingStartTime>s.tailExpiration&&1<n&&(t.effectTag|=64,o=!0,Ja(s,!1),t.expirationTime=t.childExpirationTime=n-1);s.isBackwards?(i.sibling=t.child,t.child=i):(null!==(n=s.last)?n.sibling=i:t.child=i,s.last=i)}return null!==s.tail?(0===s.tailExpiration&&(s.tailExpiration=Ur()+500),n=s.tail,s.rendering=n,s.tail=n.sibling,s.lastEffect=t.lastEffect,s.renderingStartTime=Ur(),n.sibling=null,t=Vo.current,mr(Vo,o?1&t|2:1&t),n):null}throw Error(a(156,t.tag))}function ti(e){switch(e.tag){case 1:xr(e.type)&&vr();var t=e.effectTag;return 4096&t?(e.effectTag=-4097&t|64,e):null;case 3:if(Fo(),ur(fr),ur(gr),0!=(64&(t=e.effectTag)))throw Error(a(285));return e.effectTag=-4097&t|64,e;case 5:return Bo(e),null;case 13:return ur(Vo),4096&(t=e.effectTag)?(e.effectTag=-4097&t|64,e):null;case 19:return ur(Vo),null;case 4:return Fo(),null;case 10:return ro(e),null;default:return null}}function ni(e,t){return{value:e,source:t,stack:be(t)}}za=function(e,t){for(var n=t.child;null!==n;){if(5===n.tag||6===n.tag)e.appendChild(n.stateNode);else if(4!==n.tag&&null!==n.child){n.child.return=n,n=n.child;continue}if(n===t)break;for(;null===n.sibling;){if(null===n.return||n.return===t)return;n=n.return}n.sibling.return=n.return,n=n.sibling}},Ha=function(e,t,n,s,o){var a=e.memoizedProps;if(a!==s){var i,c,l=t.stateNode;switch(Oo(Ro.current),e=null,n){case"input":a=Se(l,a),s=Se(l,s),e=[];break;case"option":a=we(l,a),s=we(l,s),e=[];break;case"select":a=r({},a,{value:void 0}),s=r({},s,{value:void 0}),e=[];break;case"textarea":a=Ce(l,a),s=Ce(l,s),e=[];break;default:"function"!=typeof a.onClick&&"function"==typeof s.onClick&&(l.onclick=dn)}for(i in on(n,s),n=null,a)if(!s.hasOwnProperty(i)&&a.hasOwnProperty(i)&&null!=a[i])if("style"===i)for(c in l=a[i])l.hasOwnProperty(c)&&(n||(n={}),n[c]="");else"dangerouslySetInnerHTML"!==i&&"children"!==i&&"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&"autoFocus"!==i&&(L.hasOwnProperty(i)?e||(e=[]):(e=e||[]).push(i,null));for(i in s){var d=s[i];if(l=null!=a?a[i]:void 0,s.hasOwnProperty(i)&&d!==l&&(null!=d||null!=l))if("style"===i)if(l){for(c in l)!l.hasOwnProperty(c)||d&&d.hasOwnProperty(c)||(n||(n={}),n[c]="");for(c in d)d.hasOwnProperty(c)&&l[c]!==d[c]&&(n||(n={}),n[c]=d[c])}else n||(e||(e=[]),e.push(i,n)),n=d;else"dangerouslySetInnerHTML"===i?(d=d?d.__html:void 0,l=l?l.__html:void 0,null!=d&&l!==d&&(e=e||[]).push(i,d)):"children"===i?l===d||"string"!=typeof d&&"number"!=typeof d||(e=e||[]).push(i,""+d):"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&(L.hasOwnProperty(i)?(null!=d&&ln(o,i),e||l===d||(e=[])):(e=e||[]).push(i,d))}n&&(e=e||[]).push("style",n),o=e,(t.updateQueue=o)&&(t.effectTag|=4)}},ja=function(e,t,n,s){n!==s&&(t.effectTag|=4)};var si="function"==typeof WeakSet?WeakSet:Set;function ri(e,t){var n=t.source,s=t.stack;null===s&&null!==n&&(s=be(n)),null!==n&&fe(n.type),t=t.value,null!==e&&1===e.tag&&fe(e.type);try{console.error(t)}catch(r){setTimeout((function(){throw r}))}}function oi(e){var t=e.ref;if(null!==t)if("function"==typeof t)try{t(null)}catch(n){kc(e,n)}else t.current=null}function ai(e,t){switch(t.tag){case 0:case 11:case 15:case 22:case 3:case 5:case 6:case 4:case 17:return;case 1:if(256&t.effectTag&&null!==e){var n=e.memoizedProps,s=e.memoizedState;t=(e=t.stateNode).getSnapshotBeforeUpdate(t.elementType===t.type?n:qr(t.type,n),s),e.__reactInternalSnapshotBeforeUpdate=t}return}throw Error(a(163))}function ii(e,t){if(null!==(t=null!==(t=t.updateQueue)?t.lastEffect:null)){var n=t=t.next;do{if((n.tag&e)===e){var s=n.destroy;n.destroy=void 0,void 0!==s&&s()}n=n.next}while(n!==t)}}function ci(e,t){if(null!==(t=null!==(t=t.updateQueue)?t.lastEffect:null)){var n=t=t.next;do{if((n.tag&e)===e){var s=n.create;n.destroy=s()}n=n.next}while(n!==t)}}function li(e,t,n){switch(n.tag){case 0:case 11:case 15:case 22:return void ci(3,n);case 1:if(e=n.stateNode,4&n.effectTag)if(null===t)e.componentDidMount();else{var s=n.elementType===n.type?t.memoizedProps:qr(n.type,t.memoizedProps);e.componentDidUpdate(s,t.memoizedState,e.__reactInternalSnapshotBeforeUpdate)}return void(null!==(t=n.updateQueue)&&fo(n,t,e));case 3:if(null!==(t=n.updateQueue)){if(e=null,null!==n.child)switch(n.child.tag){case 5:case 1:e=n.child.stateNode}fo(n,t,e)}return;case 5:return e=n.stateNode,void(null===t&&4&n.effectTag&&An(n.type,n.memoizedProps)&&e.focus());case 6:case 4:case 12:case 19:case 17:case 20:case 21:return;case 13:return void(null===n.memoizedState&&(n=n.alternate,null!==n&&(n=n.memoizedState,null!==n&&(n=n.dehydrated,null!==n&&Ft(n)))))}throw Error(a(163))}function di(e,t,n){switch("function"==typeof Cc&&Cc(t),t.tag){case 0:case 11:case 14:case 15:case 22:if(null!==(e=t.updateQueue)&&null!==(e=e.lastEffect)){var s=e.next;jr(97<n?97:n,(function(){var e=s;do{var n=e.destroy;if(void 0!==n){var r=t;try{n()}catch(o){kc(r,o)}}e=e.next}while(e!==s)}))}break;case 1:oi(t),"function"==typeof(n=t.stateNode).componentWillUnmount&&function(e,t){try{t.props=e.memoizedProps,t.state=e.memoizedState,t.componentWillUnmount()}catch(n){kc(e,n)}}(t,n);break;case 5:oi(t);break;case 4:fi(e,t,n)}}function pi(e){var t=e.alternate;e.return=null,e.child=null,e.memoizedState=null,e.updateQueue=null,e.dependencies=null,e.alternate=null,e.firstEffect=null,e.lastEffect=null,e.pendingProps=null,e.memoizedProps=null,e.stateNode=null,null!==t&&pi(t)}function ui(e){return 5===e.tag||3===e.tag||4===e.tag}function mi(e){e:{for(var t=e.return;null!==t;){if(ui(t)){var n=t;break e}t=t.return}throw Error(a(160))}switch(t=n.stateNode,n.tag){case 5:var s=!1;break;case 3:case 4:t=t.containerInfo,s=!0;break;default:throw Error(a(161))}16&n.effectTag&&(Ge(t,""),n.effectTag&=-17);e:t:for(n=e;;){for(;null===n.sibling;){if(null===n.return||ui(n.return)){n=null;break e}n=n.return}for(n.sibling.return=n.return,n=n.sibling;5!==n.tag&&6!==n.tag&&18!==n.tag;){if(2&n.effectTag)continue t;if(null===n.child||4===n.tag)continue t;n.child.return=n,n=n.child}if(!(2&n.effectTag)){n=n.stateNode;break e}}s?hi(e,n,t):gi(e,n,t)}function hi(e,t,n){var s=e.tag,r=5===s||6===s;if(r)e=r?e.stateNode:e.stateNode.instance,t?8===n.nodeType?n.parentNode.insertBefore(e,t):n.insertBefore(e,t):(8===n.nodeType?(t=n.parentNode).insertBefore(e,n):(t=n).appendChild(e),null!=(n=n._reactRootContainer)||null!==t.onclick||(t.onclick=dn));else if(4!==s&&null!==(e=e.child))for(hi(e,t,n),e=e.sibling;null!==e;)hi(e,t,n),e=e.sibling}function gi(e,t,n){var s=e.tag,r=5===s||6===s;if(r)e=r?e.stateNode:e.stateNode.instance,t?n.insertBefore(e,t):n.appendChild(e);else if(4!==s&&null!==(e=e.child))for(gi(e,t,n),e=e.sibling;null!==e;)gi(e,t,n),e=e.sibling}function fi(e,t,n){for(var s,r,o=t,i=!1;;){if(!i){i=o.return;e:for(;;){if(null===i)throw Error(a(160));switch(s=i.stateNode,i.tag){case 5:r=!1;break e;case 3:case 4:s=s.containerInfo,r=!0;break e}i=i.return}i=!0}if(5===o.tag||6===o.tag){e:for(var c=e,l=o,d=n,p=l;;)if(di(c,p,d),null!==p.child&&4!==p.tag)p.child.return=p,p=p.child;else{if(p===l)break e;for(;null===p.sibling;){if(null===p.return||p.return===l)break e;p=p.return}p.sibling.return=p.return,p=p.sibling}r?(c=s,l=o.stateNode,8===c.nodeType?c.parentNode.removeChild(l):c.removeChild(l)):s.removeChild(o.stateNode)}else if(4===o.tag){if(null!==o.child){s=o.stateNode.containerInfo,r=!0,o.child.return=o,o=o.child;continue}}else if(di(e,o,n),null!==o.child){o.child.return=o,o=o.child;continue}if(o===t)break;for(;null===o.sibling;){if(null===o.return||o.return===t)return;4===(o=o.return).tag&&(i=!1)}o.sibling.return=o.return,o=o.sibling}}function bi(e,t){switch(t.tag){case 0:case 11:case 14:case 15:case 22:return void ii(3,t);case 1:case 12:case 17:return;case 5:var n=t.stateNode;if(null!=n){var s=t.memoizedProps,r=null!==e?e.memoizedProps:s;e=t.type;var o=t.updateQueue;if(t.updateQueue=null,null!==o){for(n[In]=s,"input"===e&&"radio"===s.type&&null!=s.name&&Le(n,s),an(e,r),t=an(e,s),r=0;r<o.length;r+=2){var i=o[r],c=o[r+1];"style"===i?sn(n,c):"dangerouslySetInnerHTML"===i?Ve(n,c):"children"===i?Ge(n,c):Y(n,i,c,t)}switch(e){case"input":Me(n,s);break;case"textarea":Re(n,s);break;case"select":t=n._wrapperState.wasMultiple,n._wrapperState.wasMultiple=!!s.multiple,null!=(e=s.value)?Ie(n,!!s.multiple,e,!1):t!==!!s.multiple&&(null!=s.defaultValue?Ie(n,!!s.multiple,s.defaultValue,!0):Ie(n,!!s.multiple,s.multiple?[]:"",!1))}}}return;case 6:if(null===t.stateNode)throw Error(a(162));return void(t.stateNode.nodeValue=t.memoizedProps);case 3:return void((t=t.stateNode).hydrate&&(t.hydrate=!1,Ft(t.containerInfo)));case 13:if(n=t,null===t.memoizedState?s=!1:(s=!0,n=t.child,Wi=Ur()),null!==n)e:for(e=n;;){if(5===e.tag)o=e.stateNode,s?"function"==typeof(o=o.style).setProperty?o.setProperty("display","none","important"):o.display="none":(o=e.stateNode,r=null!=(r=e.memoizedProps.style)&&r.hasOwnProperty("display")?r.display:null,o.style.display=nn("display",r));else if(6===e.tag)e.stateNode.nodeValue=s?"":e.memoizedProps;else{if(13===e.tag&&null!==e.memoizedState&&null===e.memoizedState.dehydrated){(o=e.child.sibling).return=e,e=o;continue}if(null!==e.child){e.child.return=e,e=e.child;continue}}if(e===n)break;for(;null===e.sibling;){if(null===e.return||e.return===n)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}return void yi(t);case 19:return void yi(t)}throw Error(a(163))}function yi(e){var t=e.updateQueue;if(null!==t){e.updateQueue=null;var n=e.stateNode;null===n&&(n=e.stateNode=new si),t.forEach((function(t){var s=wc.bind(null,e,t);n.has(t)||(n.add(t),t.then(s,s))}))}}var xi="function"==typeof WeakMap?WeakMap:Map;function vi(e,t,n){(n=uo(n,null)).tag=3,n.payload={element:null};var s=t.value;return n.callback=function(){zi||(zi=!0,Hi=s),ri(e,t)},n}function Ai(e,t,n){(n=uo(n,null)).tag=3;var s=e.type.getDerivedStateFromError;if("function"==typeof s){var r=t.value;n.payload=function(){return ri(e,t),s(r)}}var o=e.stateNode;return null!==o&&"function"==typeof o.componentDidCatch&&(n.callback=function(){"function"!=typeof s&&(null===ji?ji=new Set([this]):ji.add(this),ri(e,t));var n=t.stack;this.componentDidCatch(t.value,{componentStack:null!==n?n:""})}),n}var Si,Ei=Math.ceil,Li=X.ReactCurrentDispatcher,Mi=X.ReactCurrentOwner,ki=16,_i=32,wi=0,Ii=3,Ci=4,Qi=0,Ri=null,Di=null,Ti=0,Oi=wi,Pi=null,Fi=1073741823,Ni=1073741823,Bi=null,Vi=0,Gi=!1,Wi=0,Ui=null,zi=!1,Hi=null,ji=null,$i=!1,Zi=null,Ki=90,Xi=null,Yi=0,qi=null,Ji=0;function ec(){return 0!=(48&Qi)?1073741821-(Ur()/10|0):0!==Ji?Ji:Ji=1073741821-(Ur()/10|0)}function tc(e,t,n){if(0==(2&(t=t.mode)))return 1073741823;var s=zr();if(0==(4&t))return 99===s?1073741823:1073741822;if(0!=(Qi&ki))return Ti;if(null!==n)e=Yr(e,0|n.timeoutMs||5e3,250);else switch(s){case 99:e=1073741823;break;case 98:e=Yr(e,150,100);break;case 97:case 96:e=Yr(e,5e3,250);break;case 95:e=2;break;default:throw Error(a(326))}return null!==Ri&&e===Ti&&--e,e}function nc(e,t){if(50<Yi)throw Yi=0,qi=null,Error(a(185));if(null!==(e=sc(e,t))){var n=zr();1073741823===t?0!=(8&Qi)&&0==(48&Qi)?ic(e):(oc(e),0===Qi&&Kr()):oc(e),0==(4&Qi)||98!==n&&99!==n||(null===Xi?Xi=new Map([[e,t]]):(void 0===(n=Xi.get(e))||n>t)&&Xi.set(e,t))}}function sc(e,t){e.expirationTime<t&&(e.expirationTime=t);var n=e.alternate;null!==n&&n.expirationTime<t&&(n.expirationTime=t);var s=e.return,r=null;if(null===s&&3===e.tag)r=e.stateNode;else for(;null!==s;){if(n=s.alternate,s.childExpirationTime<t&&(s.childExpirationTime=t),null!==n&&n.childExpirationTime<t&&(n.childExpirationTime=t),null===s.return&&3===s.tag){r=s.stateNode;break}s=s.return}return null!==r&&(Ri===r&&(hc(t),Oi===Ci&&Gc(r,Ti)),Wc(r,t)),r}function rc(e){var t=e.lastExpiredTime;if(0!==t)return t;if(!Vc(e,t=e.firstPendingTime))return t;var n=e.lastPingedTime;return 2>=(e=n>(e=e.nextKnownPendingLevel)?n:e)&&t!==e?0:e}function oc(e){if(0!==e.lastExpiredTime)e.callbackExpirationTime=1073741823,e.callbackPriority=99,e.callbackNode=Zr(ic.bind(null,e));else{var t=rc(e),n=e.callbackNode;if(0===t)null!==n&&(e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90);else{var s=ec();if(1073741823===t?s=99:1===t||2===t?s=95:s=0>=(s=10*(1073741821-t)-10*(1073741821-s))?99:250>=s?98:5250>=s?97:95,null!==n){var r=e.callbackPriority;if(e.callbackExpirationTime===t&&r>=s)return;n!==Pr&&_r(n)}e.callbackExpirationTime=t,e.callbackPriority=s,t=1073741823===t?Zr(ic.bind(null,e)):$r(s,ac.bind(null,e),{timeout:10*(1073741821-t)-Ur()}),e.callbackNode=t}}}function ac(e,t){if(Ji=0,t)return Uc(e,t=ec()),oc(e),null;var n=rc(e);if(0!==n){if(t=e.callbackNode,0!=(48&Qi))throw Error(a(327));if(Ec(),e===Ri&&n===Ti||dc(e,n),null!==Di){var s=Qi;Qi|=ki;for(var r=uc();;)try{fc();break}catch(c){pc(e,c)}if(so(),Qi=s,Li.current=r,1===Oi)throw t=Pi,dc(e,n),Gc(e,n),oc(e),t;if(null===Di)switch(r=e.finishedWork=e.current.alternate,e.finishedExpirationTime=n,s=Oi,Ri=null,s){case wi:case 1:throw Error(a(345));case 2:Uc(e,2<n?2:n);break;case Ii:if(Gc(e,n),n===(s=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=xc(r)),1073741823===Fi&&10<(r=Wi+500-Ur())){if(Gi){var o=e.lastPingedTime;if(0===o||o>=n){e.lastPingedTime=n,dc(e,n);break}}if(0!==(o=rc(e))&&o!==n)break;if(0!==s&&s!==n){e.lastPingedTime=s;break}e.timeoutHandle=En(vc.bind(null,e),r);break}vc(e);break;case Ci:if(Gc(e,n),n===(s=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=xc(r)),Gi&&(0===(r=e.lastPingedTime)||r>=n)){e.lastPingedTime=n,dc(e,n);break}if(0!==(r=rc(e))&&r!==n)break;if(0!==s&&s!==n){e.lastPingedTime=s;break}if(1073741823!==Ni?s=10*(1073741821-Ni)-Ur():1073741823===Fi?s=0:(s=10*(1073741821-Fi)-5e3,0>(s=(r=Ur())-s)&&(s=0),(n=10*(1073741821-n)-r)<(s=(120>s?120:480>s?480:1080>s?1080:1920>s?1920:3e3>s?3e3:4320>s?4320:1960*Ei(s/1960))-s)&&(s=n)),10<s){e.timeoutHandle=En(vc.bind(null,e),s);break}vc(e);break;case 5:if(1073741823!==Fi&&null!==Bi){o=Fi;var i=Bi;if(0>=(s=0|i.busyMinDurationMs)?s=0:(r=0|i.busyDelayMs,s=(o=Ur()-(10*(1073741821-o)-(0|i.timeoutMs||5e3)))<=r?0:r+s-o),10<s){Gc(e,n),e.timeoutHandle=En(vc.bind(null,e),s);break}}vc(e);break;default:throw Error(a(329))}if(oc(e),e.callbackNode===t)return ac.bind(null,e)}}return null}function ic(e){var t=e.lastExpiredTime;if(t=0!==t?t:1073741823,0!=(48&Qi))throw Error(a(327));if(Ec(),e===Ri&&t===Ti||dc(e,t),null!==Di){var n=Qi;Qi|=ki;for(var s=uc();;)try{gc();break}catch(r){pc(e,r)}if(so(),Qi=n,Li.current=s,1===Oi)throw n=Pi,dc(e,t),Gc(e,t),oc(e),n;if(null!==Di)throw Error(a(261));e.finishedWork=e.current.alternate,e.finishedExpirationTime=t,Ri=null,vc(e),oc(e)}return null}function cc(e,t){var n=Qi;Qi|=1;try{return e(t)}finally{0===(Qi=n)&&Kr()}}function lc(e,t){var n=Qi;Qi&=-2,Qi|=8;try{return e(t)}finally{0===(Qi=n)&&Kr()}}function dc(e,t){e.finishedWork=null,e.finishedExpirationTime=0;var n=e.timeoutHandle;if(-1!==n&&(e.timeoutHandle=-1,Ln(n)),null!==Di)for(n=Di.return;null!==n;){var s=n;switch(s.tag){case 1:null!=(s=s.type.childContextTypes)&&vr();break;case 3:Fo(),ur(fr),ur(gr);break;case 5:Bo(s);break;case 4:Fo();break;case 13:case 19:ur(Vo);break;case 10:ro(s)}n=n.return}Ri=e,Di=Tc(e.current,null),Ti=t,Oi=wi,Pi=null,Ni=Fi=1073741823,Bi=null,Vi=0,Gi=!1}function pc(e,t){for(;;){try{if(so(),Uo.current=va,Ko)for(var n=jo.memoizedState;null!==n;){var s=n.queue;null!==s&&(s.pending=null),n=n.next}if(Ho=0,Zo=$o=jo=null,Ko=!1,null===Di||null===Di.return)return Oi=1,Pi=t,Di=null;e:{var r=e,o=Di.return,a=Di,i=t;if(t=Ti,a.effectTag|=2048,a.firstEffect=a.lastEffect=null,null!==i&&"object"==typeof i&&"function"==typeof i.then){var c=i;if(0==(2&a.mode)){var l=a.alternate;l?(a.updateQueue=l.updateQueue,a.memoizedState=l.memoizedState,a.expirationTime=l.expirationTime):(a.updateQueue=null,a.memoizedState=null)}var d=0!=(1&Vo.current),p=o;do{var u;if(u=13===p.tag){var m=p.memoizedState;if(null!==m)u=null!==m.dehydrated;else{var h=p.memoizedProps;u=void 0!==h.fallback&&(!0!==h.unstable_avoidThisFallback||!d)}}if(u){var g=p.updateQueue;if(null===g){var f=new Set;f.add(c),p.updateQueue=f}else g.add(c);if(0==(2&p.mode)){if(p.effectTag|=64,a.effectTag&=-2981,1===a.tag)if(null===a.alternate)a.tag=17;else{var b=uo(1073741823,null);b.tag=2,mo(a,b)}a.expirationTime=1073741823;break e}i=void 0,a=t;var y=r.pingCache;if(null===y?(y=r.pingCache=new xi,i=new Set,y.set(c,i)):void 0===(i=y.get(c))&&(i=new Set,y.set(c,i)),!i.has(a)){i.add(a);var x=_c.bind(null,r,c,a);c.then(x,x)}p.effectTag|=4096,p.expirationTime=t;break e}p=p.return}while(null!==p);i=Error((fe(a.type)||"A React component")+" suspended while rendering, but no fallback UI was specified.\n\nAdd a <Suspense fallback=...> component higher in the tree to provide a loading indicator or placeholder to display."+be(a))}5!==Oi&&(Oi=2),i=ni(i,a),p=o;do{switch(p.tag){case 3:c=i,p.effectTag|=4096,p.expirationTime=t,ho(p,vi(p,c,t));break e;case 1:c=i;var v=p.type,A=p.stateNode;if(0==(64&p.effectTag)&&("function"==typeof v.getDerivedStateFromError||null!==A&&"function"==typeof A.componentDidCatch&&(null===ji||!ji.has(A)))){p.effectTag|=4096,p.expirationTime=t,ho(p,Ai(p,c,t));break e}}p=p.return}while(null!==p)}Di=yc(Di)}catch(S){t=S;continue}break}}function uc(){var e=Li.current;return Li.current=va,null===e?va:e}function mc(e,t){e<Fi&&2<e&&(Fi=e),null!==t&&e<Ni&&2<e&&(Ni=e,Bi=t)}function hc(e){e>Vi&&(Vi=e)}function gc(){for(;null!==Di;)Di=bc(Di)}function fc(){for(;null!==Di&&!Fr();)Di=bc(Di)}function bc(e){var t=Si(e.alternate,e,Ti);return e.memoizedProps=e.pendingProps,null===t&&(t=yc(e)),Mi.current=null,t}function yc(e){Di=e;do{var t=Di.alternate;if(e=Di.return,0==(2048&Di.effectTag)){if(t=ei(t,Di,Ti),1===Ti||1!==Di.childExpirationTime){for(var n=0,s=Di.child;null!==s;){var r=s.expirationTime,o=s.childExpirationTime;r>n&&(n=r),o>n&&(n=o),s=s.sibling}Di.childExpirationTime=n}if(null!==t)return t;null!==e&&0==(2048&e.effectTag)&&(null===e.firstEffect&&(e.firstEffect=Di.firstEffect),null!==Di.lastEffect&&(null!==e.lastEffect&&(e.lastEffect.nextEffect=Di.firstEffect),e.lastEffect=Di.lastEffect),1<Di.effectTag&&(null!==e.lastEffect?e.lastEffect.nextEffect=Di:e.firstEffect=Di,e.lastEffect=Di))}else{if(null!==(t=ti(Di)))return t.effectTag&=2047,t;null!==e&&(e.firstEffect=e.lastEffect=null,e.effectTag|=2048)}if(null!==(t=Di.sibling))return t;Di=e}while(null!==Di);return Oi===wi&&(Oi=5),null}function xc(e){var t=e.expirationTime;return t>(e=e.childExpirationTime)?t:e}function vc(e){var t=zr();return jr(99,Ac.bind(null,e,t)),null}function Ac(e,t){do{Ec()}while(null!==Zi);if(0!=(48&Qi))throw Error(a(327));var n=e.finishedWork,s=e.finishedExpirationTime;if(null===n)return null;if(e.finishedWork=null,e.finishedExpirationTime=0,n===e.current)throw Error(a(177));e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90,e.nextKnownPendingLevel=0;var r=xc(n);if(e.firstPendingTime=r,s<=e.lastSuspendedTime?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:s<=e.firstSuspendedTime&&(e.firstSuspendedTime=s-1),s<=e.lastPingedTime&&(e.lastPingedTime=0),s<=e.lastExpiredTime&&(e.lastExpiredTime=0),e===Ri&&(Di=Ri=null,Ti=0),1<n.effectTag?null!==n.lastEffect?(n.lastEffect.nextEffect=n,r=n.firstEffect):r=n:r=n.firstEffect,null!==r){var o=Qi;Qi|=_i,Mi.current=null,xn=$t;var i=gn();if(fn(i)){if("selectionStart"in i)var c={start:i.selectionStart,end:i.selectionEnd};else e:{var l=(c=(c=i.ownerDocument)&&c.defaultView||window).getSelection&&c.getSelection();if(l&&0!==l.rangeCount){c=l.anchorNode;var d=l.anchorOffset,p=l.focusNode;l=l.focusOffset;try{c.nodeType,p.nodeType}catch(k){c=null;break e}var u=0,m=-1,h=-1,g=0,f=0,b=i,y=null;t:for(;;){for(var x;b!==c||0!==d&&3!==b.nodeType||(m=u+d),b!==p||0!==l&&3!==b.nodeType||(h=u+l),3===b.nodeType&&(u+=b.nodeValue.length),null!==(x=b.firstChild);)y=b,b=x;for(;;){if(b===i)break t;if(y===c&&++g===d&&(m=u),y===p&&++f===l&&(h=u),null!==(x=b.nextSibling))break;y=(b=y).parentNode}b=x}c=-1===m||-1===h?null:{start:m,end:h}}else c=null}c=c||{start:0,end:0}}else c=null;vn={activeElementDetached:null,focusedElem:i,selectionRange:c},$t=!1,Ui=r;do{try{Sc()}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);Ui=r;do{try{for(i=e,c=t;null!==Ui;){var v=Ui.effectTag;if(16&v&&Ge(Ui.stateNode,""),128&v){var A=Ui.alternate;if(null!==A){var S=A.ref;null!==S&&("function"==typeof S?S(null):S.current=null)}}switch(1038&v){case 2:mi(Ui),Ui.effectTag&=-3;break;case 6:mi(Ui),Ui.effectTag&=-3,bi(Ui.alternate,Ui);break;case 1024:Ui.effectTag&=-1025;break;case 1028:Ui.effectTag&=-1025,bi(Ui.alternate,Ui);break;case 4:bi(Ui.alternate,Ui);break;case 8:fi(i,d=Ui,c),pi(d)}Ui=Ui.nextEffect}}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);if(S=vn,A=gn(),v=S.focusedElem,c=S.selectionRange,A!==v&&v&&v.ownerDocument&&hn(v.ownerDocument.documentElement,v)){null!==c&&fn(v)&&(A=c.start,void 0===(S=c.end)&&(S=A),"selectionStart"in v?(v.selectionStart=A,v.selectionEnd=Math.min(S,v.value.length)):(S=(A=v.ownerDocument||document)&&A.defaultView||window).getSelection&&(S=S.getSelection(),d=v.textContent.length,i=Math.min(c.start,d),c=void 0===c.end?i:Math.min(c.end,d),!S.extend&&i>c&&(d=c,c=i,i=d),d=mn(v,i),p=mn(v,c),d&&p&&(1!==S.rangeCount||S.anchorNode!==d.node||S.anchorOffset!==d.offset||S.focusNode!==p.node||S.focusOffset!==p.offset)&&((A=A.createRange()).setStart(d.node,d.offset),S.removeAllRanges(),i>c?(S.addRange(A),S.extend(p.node,p.offset)):(A.setEnd(p.node,p.offset),S.addRange(A))))),A=[];for(S=v;S=S.parentNode;)1===S.nodeType&&A.push({element:S,left:S.scrollLeft,top:S.scrollTop});for("function"==typeof v.focus&&v.focus(),v=0;v<A.length;v++)(S=A[v]).element.scrollLeft=S.left,S.element.scrollTop=S.top}$t=!!xn,vn=xn=null,e.current=n,Ui=r;do{try{for(v=e;null!==Ui;){var E=Ui.effectTag;if(36&E&&li(v,Ui.alternate,Ui),128&E){A=void 0;var L=Ui.ref;if(null!==L){var M=Ui.stateNode;Ui.tag,A=M,"function"==typeof L?L(A):L.current=A}}Ui=Ui.nextEffect}}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);Ui=null,Nr(),Qi=o}else e.current=n;if($i)$i=!1,Zi=e,Ki=t;else for(Ui=r;null!==Ui;)t=Ui.nextEffect,Ui.nextEffect=null,Ui=t;if(0===(t=e.firstPendingTime)&&(ji=null),1073741823===t?e===qi?Yi++:(Yi=0,qi=e):Yi=0,"function"==typeof Ic&&Ic(n.stateNode,s),oc(e),zi)throw zi=!1,e=Hi,Hi=null,e;return 0!=(8&Qi)||Kr(),null}function Sc(){for(;null!==Ui;){var e=Ui.effectTag;0!=(256&e)&&ai(Ui.alternate,Ui),0==(512&e)||$i||($i=!0,$r(97,(function(){return Ec(),null}))),Ui=Ui.nextEffect}}function Ec(){if(90!==Ki){var e=97<Ki?97:Ki;return Ki=90,jr(e,Lc)}}function Lc(){if(null===Zi)return!1;var e=Zi;if(Zi=null,0!=(48&Qi))throw Error(a(331));var t=Qi;for(Qi|=_i,e=e.current.firstEffect;null!==e;){try{var n=e;if(0!=(512&n.effectTag))switch(n.tag){case 0:case 11:case 15:case 22:ii(5,n),ci(5,n)}}catch(s){if(null===e)throw Error(a(330));kc(e,s)}n=e.nextEffect,e.nextEffect=null,e=n}return Qi=t,Kr(),!0}function Mc(e,t,n){mo(e,t=vi(e,t=ni(n,t),1073741823)),null!==(e=sc(e,1073741823))&&oc(e)}function kc(e,t){if(3===e.tag)Mc(e,e,t);else for(var n=e.return;null!==n;){if(3===n.tag){Mc(n,e,t);break}if(1===n.tag){var s=n.stateNode;if("function"==typeof n.type.getDerivedStateFromError||"function"==typeof s.componentDidCatch&&(null===ji||!ji.has(s))){mo(n,e=Ai(n,e=ni(t,e),1073741823)),null!==(n=sc(n,1073741823))&&oc(n);break}}n=n.return}}function _c(e,t,n){var s=e.pingCache;null!==s&&s.delete(t),Ri===e&&Ti===n?Oi===Ci||Oi===Ii&&1073741823===Fi&&Ur()-Wi<500?dc(e,Ti):Gi=!0:Vc(e,n)&&(0!==(t=e.lastPingedTime)&&t<n||(e.lastPingedTime=n,oc(e)))}function wc(e,t){var n=e.stateNode;null!==n&&n.delete(t),0===(t=0)&&(t=tc(t=ec(),e,null)),null!==(e=sc(e,t))&&oc(e)}Si=function(e,t,n){var s=t.expirationTime;if(null!==e){var r=t.pendingProps;if(e.memoizedProps!==r||fr.current)Ta=!0;else{if(s<n){switch(Ta=!1,t.tag){case 3:Ua(t),Ra();break;case 5:if(No(t),4&t.mode&&1!==n&&r.hidden)return t.expirationTime=t.childExpirationTime=1,null;break;case 1:xr(t.type)&&Er(t);break;case 4:Po(t,t.stateNode.containerInfo);break;case 10:s=t.memoizedProps.value,r=t.type._context,mr(Jr,r._currentValue),r._currentValue=s;break;case 13:if(null!==t.memoizedState)return 0!==(s=t.child.childExpirationTime)&&s>=n?Za(e,t,n):(mr(Vo,1&Vo.current),null!==(t=qa(e,t,n))?t.sibling:null);mr(Vo,1&Vo.current);break;case 19:if(s=t.childExpirationTime>=n,0!=(64&e.effectTag)){if(s)return Ya(e,t,n);t.effectTag|=64}if(null!==(r=t.memoizedState)&&(r.rendering=null,r.tail=null),mr(Vo,Vo.current),!s)return null}return qa(e,t,n)}Ta=!1}}else Ta=!1;switch(t.expirationTime=0,t.tag){case 2:if(s=t.type,null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),e=t.pendingProps,r=yr(t,gr.current),ao(t,n),r=qo(null,t,s,e,r,n),t.effectTag|=1,"object"==typeof r&&null!==r&&"function"==typeof r.render&&void 0===r.$$typeof){if(t.tag=1,t.memoizedState=null,t.updateQueue=null,xr(s)){var o=!0;Er(t)}else o=!1;t.memoizedState=null!==r.state&&void 0!==r.state?r.state:null,lo(t);var i=s.getDerivedStateFromProps;"function"==typeof i&&xo(t,s,i,e),r.updater=vo,t.stateNode=r,r._reactInternalFiber=t,Lo(t,s,e,n),t=Wa(null,t,s,!0,o,n)}else t.tag=0,Oa(null,t,r,n),t=t.child;return t;case 16:e:{if(r=t.elementType,null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),e=t.pendingProps,function(e){if(-1===e._status){e._status=0;var t=e._ctor;t=t(),e._result=t,t.then((function(t){0===e._status&&(t=t.default,e._status=1,e._result=t)}),(function(t){0===e._status&&(e._status=2,e._result=t)}))}}(r),1!==r._status)throw r._result;switch(r=r._result,t.type=r,o=t.tag=function(e){if("function"==typeof e)return Dc(e)?1:0;if(null!=e){if((e=e.$$typeof)===ce)return 11;if(e===pe)return 14}return 2}(r),e=qr(r,e),o){case 0:t=Va(null,t,r,e,n);break e;case 1:t=Ga(null,t,r,e,n);break e;case 11:t=Pa(null,t,r,e,n);break e;case 14:t=Fa(null,t,r,qr(r.type,e),s,n);break e}throw Error(a(306,r,""))}return t;case 0:return s=t.type,r=t.pendingProps,Va(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 1:return s=t.type,r=t.pendingProps,Ga(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 3:if(Ua(t),s=t.updateQueue,null===e||null===s)throw Error(a(282));if(s=t.pendingProps,r=null!==(r=t.memoizedState)?r.element:null,po(e,t),go(t,s,null,n),(s=t.memoizedState.element)===r)Ra(),t=qa(e,t,n);else{if((r=t.stateNode.hydrate)&&(Ma=Mn(t.stateNode.containerInfo.firstChild),La=t,r=ka=!0),r)for(n=Co(t,null,s,n),t.child=n;n;)n.effectTag=-3&n.effectTag|1024,n=n.sibling;else Oa(e,t,s,n),Ra();t=t.child}return t;case 5:return No(t),null===e&&Ia(t),s=t.type,r=t.pendingProps,o=null!==e?e.memoizedProps:null,i=r.children,Sn(s,r)?i=null:null!==o&&Sn(s,o)&&(t.effectTag|=16),Ba(e,t),4&t.mode&&1!==n&&r.hidden?(t.expirationTime=t.childExpirationTime=1,t=null):(Oa(e,t,i,n),t=t.child),t;case 6:return null===e&&Ia(t),null;case 13:return Za(e,t,n);case 4:return Po(t,t.stateNode.containerInfo),s=t.pendingProps,null===e?t.child=Io(t,null,s,n):Oa(e,t,s,n),t.child;case 11:return s=t.type,r=t.pendingProps,Pa(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 7:return Oa(e,t,t.pendingProps,n),t.child;case 8:case 12:return Oa(e,t,t.pendingProps.children,n),t.child;case 10:e:{s=t.type._context,r=t.pendingProps,i=t.memoizedProps,o=r.value;var c=t.type._context;if(mr(Jr,c._currentValue),c._currentValue=o,null!==i)if(c=i.value,0===(o=Gs(c,o)?0:0|("function"==typeof s._calculateChangedBits?s._calculateChangedBits(c,o):1073741823))){if(i.children===r.children&&!fr.current){t=qa(e,t,n);break e}}else for(null!==(c=t.child)&&(c.return=t);null!==c;){var l=c.dependencies;if(null!==l){i=c.child;for(var d=l.firstContext;null!==d;){if(d.context===s&&0!=(d.observedBits&o)){1===c.tag&&((d=uo(n,null)).tag=2,mo(c,d)),c.expirationTime<n&&(c.expirationTime=n),null!==(d=c.alternate)&&d.expirationTime<n&&(d.expirationTime=n),oo(c.return,n),l.expirationTime<n&&(l.expirationTime=n);break}d=d.next}}else i=10===c.tag&&c.type===t.type?null:c.child;if(null!==i)i.return=c;else for(i=c;null!==i;){if(i===t){i=null;break}if(null!==(c=i.sibling)){c.return=i.return,i=c;break}i=i.return}c=i}Oa(e,t,r.children,n),t=t.child}return t;case 9:return r=t.type,s=(o=t.pendingProps).children,ao(t,n),s=s(r=io(r,o.unstable_observedBits)),t.effectTag|=1,Oa(e,t,s,n),t.child;case 14:return o=qr(r=t.type,t.pendingProps),Fa(e,t,r,o=qr(r.type,o),s,n);case 15:return Na(e,t,t.type,t.pendingProps,s,n);case 17:return s=t.type,r=t.pendingProps,r=t.elementType===s?r:qr(s,r),null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),t.tag=1,xr(s)?(e=!0,Er(t)):e=!1,ao(t,n),So(t,s,r),Lo(t,s,r,n),Wa(null,t,s,!0,e,n);case 19:return Ya(e,t,n)}throw Error(a(156,t.tag))};var Ic=null,Cc=null;function Qc(e,t,n,s){this.tag=e,this.key=n,this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null,this.index=0,this.ref=null,this.pendingProps=t,this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null,this.mode=s,this.effectTag=0,this.lastEffect=this.firstEffect=this.nextEffect=null,this.childExpirationTime=this.expirationTime=0,this.alternate=null}function Rc(e,t,n,s){return new Qc(e,t,n,s)}function Dc(e){return!(!(e=e.prototype)||!e.isReactComponent)}function Tc(e,t){var n=e.alternate;return null===n?((n=Rc(e.tag,t,e.key,e.mode)).elementType=e.elementType,n.type=e.type,n.stateNode=e.stateNode,n.alternate=e,e.alternate=n):(n.pendingProps=t,n.effectTag=0,n.nextEffect=null,n.firstEffect=null,n.lastEffect=null),n.childExpirationTime=e.childExpirationTime,n.expirationTime=e.expirationTime,n.child=e.child,n.memoizedProps=e.memoizedProps,n.memoizedState=e.memoizedState,n.updateQueue=e.updateQueue,t=e.dependencies,n.dependencies=null===t?null:{expirationTime:t.expirationTime,firstContext:t.firstContext,responders:t.responders},n.sibling=e.sibling,n.index=e.index,n.ref=e.ref,n}function Oc(e,t,n,s,r,o){var i=2;if(s=e,"function"==typeof e)Dc(e)&&(i=1);else if("string"==typeof e)i=5;else e:switch(e){case ne:return Pc(n.children,r,o,t);case ie:i=8,r|=7;break;case se:i=8,r|=1;break;case re:return(e=Rc(12,n,t,8|r)).elementType=re,e.type=re,e.expirationTime=o,e;case le:return(e=Rc(13,n,t,r)).type=le,e.elementType=le,e.expirationTime=o,e;case de:return(e=Rc(19,n,t,r)).elementType=de,e.expirationTime=o,e;default:if("object"==typeof e&&null!==e)switch(e.$$typeof){case oe:i=10;break e;case ae:i=9;break e;case ce:i=11;break e;case pe:i=14;break e;case ue:i=16,s=null;break e;case me:i=22;break e}throw Error(a(130,null==e?e:typeof e,""))}return(t=Rc(i,n,t,r)).elementType=e,t.type=s,t.expirationTime=o,t}function Pc(e,t,n,s){return(e=Rc(7,e,s,t)).expirationTime=n,e}function Fc(e,t,n){return(e=Rc(6,e,null,t)).expirationTime=n,e}function Nc(e,t,n){return(t=Rc(4,null!==e.children?e.children:[],e.key,t)).expirationTime=n,t.stateNode={containerInfo:e.containerInfo,pendingChildren:null,implementation:e.implementation},t}function Bc(e,t,n){this.tag=t,this.current=null,this.containerInfo=e,this.pingCache=this.pendingChildren=null,this.finishedExpirationTime=0,this.finishedWork=null,this.timeoutHandle=-1,this.pendingContext=this.context=null,this.hydrate=n,this.callbackNode=null,this.callbackPriority=90,this.lastExpiredTime=this.lastPingedTime=this.nextKnownPendingLevel=this.lastSuspendedTime=this.firstSuspendedTime=this.firstPendingTime=0}function Vc(e,t){var n=e.firstSuspendedTime;return e=e.lastSuspendedTime,0!==n&&n>=t&&e<=t}function Gc(e,t){var n=e.firstSuspendedTime,s=e.lastSuspendedTime;n<t&&(e.firstSuspendedTime=t),(s>t||0===n)&&(e.lastSuspendedTime=t),t<=e.lastPingedTime&&(e.lastPingedTime=0),t<=e.lastExpiredTime&&(e.lastExpiredTime=0)}function Wc(e,t){t>e.firstPendingTime&&(e.firstPendingTime=t);var n=e.firstSuspendedTime;0!==n&&(t>=n?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:t>=e.lastSuspendedTime&&(e.lastSuspendedTime=t+1),t>e.nextKnownPendingLevel&&(e.nextKnownPendingLevel=t))}function Uc(e,t){var n=e.lastExpiredTime;(0===n||n>t)&&(e.lastExpiredTime=t)}function zc(e,t,n,s){var r=t.current,o=ec(),i=bo.suspense;o=tc(o,r,i);e:if(n){t:{if(et(n=n._reactInternalFiber)!==n||1!==n.tag)throw Error(a(170));var c=n;do{switch(c.tag){case 3:c=c.stateNode.context;break t;case 1:if(xr(c.type)){c=c.stateNode.__reactInternalMemoizedMergedChildContext;break t}}c=c.return}while(null!==c);throw Error(a(171))}if(1===n.tag){var l=n.type;if(xr(l)){n=Sr(n,l,c);break e}}n=c}else n=hr;return null===t.context?t.context=n:t.pendingContext=n,(t=uo(o,i)).payload={element:e},null!==(s=void 0===s?null:s)&&(t.callback=s),mo(r,t),nc(r,o),o}function Hc(e){return(e=e.current).child?(e.child.tag,e.child.stateNode):null}function jc(e,t){null!==(e=e.memoizedState)&&null!==e.dehydrated&&e.retryTime<t&&(e.retryTime=t)}function $c(e,t){jc(e,t),(e=e.alternate)&&jc(e,t)}function Zc(e,t,n){var s=new Bc(e,t,n=null!=n&&!0===n.hydrate),r=Rc(3,null,null,2===t?7:1===t?3:0);s.current=r,r.stateNode=s,lo(r),e[Cn]=s.current,n&&0!==t&&function(e,t){var n=Je(t);_t.forEach((function(e){gt(e,t,n)})),wt.forEach((function(e){gt(e,t,n)}))}(0,9===e.nodeType?e:e.ownerDocument),this._internalRoot=s}function Kc(e){return!(!e||1!==e.nodeType&&9!==e.nodeType&&11!==e.nodeType&&(8!==e.nodeType||" react-mount-point-unstable "!==e.nodeValue))}function Xc(e,t,n,s,r){var o=n._reactRootContainer;if(o){var a=o._internalRoot;if("function"==typeof r){var i=r;r=function(){var e=Hc(a);i.call(e)}}zc(t,a,e,r)}else{if(o=n._reactRootContainer=function(e,t){if(t||(t=!(!(t=e?9===e.nodeType?e.documentElement:e.firstChild:null)||1!==t.nodeType||!t.hasAttribute("data-reactroot"))),!t)for(var n;n=e.lastChild;)e.removeChild(n);return new Zc(e,0,t?{hydrate:!0}:void 0)}(n,s),a=o._internalRoot,"function"==typeof r){var c=r;r=function(){var e=Hc(a);c.call(e)}}lc((function(){zc(t,a,e,r)}))}return Hc(a)}function Yc(e,t,n){var s=3<arguments.length&&void 0!==arguments[3]?arguments[3]:null;return{$$typeof:te,key:null==s?null:""+s,children:e,containerInfo:t,implementation:n}}function qc(e,t){var n=2<arguments.length&&void 0!==arguments[2]?arguments[2]:null;if(!Kc(t))throw Error(a(200));return Yc(e,t,null,n)}Zc.prototype.render=function(e){zc(e,this._internalRoot,null,null)},Zc.prototype.unmount=function(){var e=this._internalRoot,t=e.containerInfo;zc(null,e,null,(function(){t[Cn]=null}))},ft=function(e){if(13===e.tag){var t=Yr(ec(),150,100);nc(e,t),$c(e,t)}},bt=function(e){13===e.tag&&(nc(e,3),$c(e,3))},yt=function(e){if(13===e.tag){var t=ec();nc(e,t=tc(t,e,null)),$c(e,t)}},w=function(e,t,n){switch(t){case"input":if(Me(e,n),t=n.name,"radio"===n.type&&null!=t){for(n=e;n.parentNode;)n=n.parentNode;for(n=n.querySelectorAll("input[name="+JSON.stringify(""+t)+'][type="radio"]'),t=0;t<n.length;t++){var s=n[t];if(s!==e&&s.form===e.form){var r=Tn(s);if(!r)throw Error(a(90));Ae(s),Me(s,r)}}}break;case"textarea":Re(e,n);break;case"select":null!=(t=n.value)&&Ie(e,!!n.multiple,t,!1)}},T=cc,O=function(e,t,n,s,r){var o=Qi;Qi|=4;try{return jr(98,e.bind(null,t,n,s,r))}finally{0===(Qi=o)&&Kr()}},P=function(){0==(49&Qi)&&(function(){if(null!==Xi){var e=Xi;Xi=null,e.forEach((function(e,t){Uc(t,e),oc(t)})),Kr()}}(),Ec())},F=function(e,t){var n=Qi;Qi|=2;try{return e(t)}finally{0===(Qi=n)&&Kr()}};var Jc={Events:[Rn,Dn,Tn,k,E,Gn,function(e){ot(e,Vn)},R,D,qt,ct,Ec,{current:!1}]};!function(e){var t=e.findFiberByHostInstance;(function(e){if("undefined"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__)return!1;var t=__REACT_DEVTOOLS_GLOBAL_HOOK__;if(t.isDisabled||!t.supportsFiber)return!0;try{var n=t.inject(e);Ic=function(e){try{t.onCommitFiberRoot(n,e,void 0,64==(64&e.current.effectTag))}catch(s){}},Cc=function(e){try{t.onCommitFiberUnmount(n,e)}catch(s){}}}catch(s){}})(r({},e,{overrideHookState:null,overrideProps:null,setSuspenseHandler:null,scheduleUpdate:null,currentDispatcherRef:X.ReactCurrentDispatcher,findHostInstanceByFiber:function(e){return null===(e=st(e))?null:e.stateNode},findFiberByHostInstance:function(e){return t?t(e):null},findHostInstancesForRefresh:null,scheduleRefresh:null,scheduleRoot:null,setRefreshHandler:null,getCurrentFiber:null}))}({findFiberByHostInstance:Qn,bundleType:0,version:"16.14.0",rendererPackageName:"react-dom"}),t.createPortal=qc,t.hydrate=function(e,t,n){if(!Kc(t))throw Error(a(200));return Xc(null,e,t,!0,n)}},73935:(e,t,n)=>{"use strict";!function e(){if("undefined"!=typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(t){console.error(t)}}(),e.exports=n(64448)},69590:e=>{var t="undefined"!=typeof Element,n="function"==typeof Map,s="function"==typeof Set,r="function"==typeof ArrayBuffer&&!!ArrayBuffer.isView;function o(e,a){if(e===a)return!0;if(e&&a&&"object"==typeof e&&"object"==typeof a){if(e.constructor!==a.constructor)return!1;var i,c,l,d;if(Array.isArray(e)){if((i=e.length)!=a.length)return!1;for(c=i;0!=c--;)if(!o(e[c],a[c]))return!1;return!0}if(n&&e instanceof Map&&a instanceof Map){if(e.size!==a.size)return!1;for(d=e.entries();!(c=d.next()).done;)if(!a.has(c.value[0]))return!1;for(d=e.entries();!(c=d.next()).done;)if(!o(c.value[1],a.get(c.value[0])))return!1;return!0}if(s&&e instanceof Set&&a instanceof Set){if(e.size!==a.size)return!1;for(d=e.entries();!(c=d.next()).done;)if(!a.has(c.value[0]))return!1;return!0}if(r&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(a)){if((i=e.length)!=a.length)return!1;for(c=i;0!=c--;)if(e[c]!==a[c])return!1;return!0}if(e.constructor===RegExp)return e.source===a.source&&e.flags===a.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===a.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===a.toString();if((i=(l=Object.keys(e)).length)!==Object.keys(a).length)return!1;for(c=i;0!=c--;)if(!Object.prototype.hasOwnProperty.call(a,l[c]))return!1;if(t&&e instanceof Element)return!1;for(c=i;0!=c--;)if(("_owner"!==l[c]&&"__v"!==l[c]&&"__o"!==l[c]||!e.$$typeof)&&!o(e[l[c]],a[l[c]]))return!1;return!0}return e!=e&&a!=a}e.exports=function(e,t){try{return o(e,t)}catch(n){if((n.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw n}}},70405:(e,t,n)=>{"use strict";n.d(t,{B6:()=>H,ql:()=>J});var s=n(67294),r=n(45697),o=n.n(r),a=n(69590),i=n.n(a),c=n(41143),l=n.n(c),d=n(96774),p=n.n(d);function u(){return u=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},u.apply(this,arguments)}function m(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,h(e,t)}function h(e,t){return h=Object.setPrototypeOf||function(e,t){return e.__proto__=t,e},h(e,t)}function g(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)t.indexOf(n=o[s])>=0||(r[n]=e[n]);return r}var f={BASE:"base",BODY:"body",HEAD:"head",HTML:"html",LINK:"link",META:"meta",NOSCRIPT:"noscript",SCRIPT:"script",STYLE:"style",TITLE:"title",FRAGMENT:"Symbol(react.fragment)"},b={rel:["amphtml","canonical","alternate"]},y={type:["application/ld+json"]},x={charset:"",name:["robots","description"],property:["og:type","og:title","og:url","og:image","og:image:alt","og:description","twitter:url","twitter:title","twitter:description","twitter:image","twitter:image:alt","twitter:card","twitter:site"]},v=Object.keys(f).map((function(e){return f[e]})),A={accesskey:"accessKey",charset:"charSet",class:"className",contenteditable:"contentEditable",contextmenu:"contextMenu","http-equiv":"httpEquiv",itemprop:"itemProp",tabindex:"tabIndex"},S=Object.keys(A).reduce((function(e,t){return e[A[t]]=t,e}),{}),E=function(e,t){for(var n=e.length-1;n>=0;n-=1){var s=e[n];if(Object.prototype.hasOwnProperty.call(s,t))return s[t]}return null},L=function(e){var t=E(e,f.TITLE),n=E(e,"titleTemplate");if(Array.isArray(t)&&(t=t.join("")),n&&t)return n.replace(/%s/g,(function(){return t}));var s=E(e,"defaultTitle");return t||s||void 0},M=function(e){return E(e,"onChangeClientState")||function(){}},k=function(e,t){return t.filter((function(t){return void 0!==t[e]})).map((function(t){return t[e]})).reduce((function(e,t){return u({},e,t)}),{})},_=function(e,t){return t.filter((function(e){return void 0!==e[f.BASE]})).map((function(e){return e[f.BASE]})).reverse().reduce((function(t,n){if(!t.length)for(var s=Object.keys(n),r=0;r<s.length;r+=1){var o=s[r].toLowerCase();if(-1!==e.indexOf(o)&&n[o])return t.concat(n)}return t}),[])},w=function(e,t,n){var s={};return n.filter((function(t){return!!Array.isArray(t[e])||(void 0!==t[e]&&console&&"function"==typeof console.warn&&console.warn("Helmet: "+e+' should be of type "Array". Instead found type "'+typeof t[e]+'"'),!1)})).map((function(t){return t[e]})).reverse().reduce((function(e,n){var r={};n.filter((function(e){for(var n,o=Object.keys(e),a=0;a<o.length;a+=1){var i=o[a],c=i.toLowerCase();-1===t.indexOf(c)||"rel"===n&&"canonical"===e[n].toLowerCase()||"rel"===c&&"stylesheet"===e[c].toLowerCase()||(n=c),-1===t.indexOf(i)||"innerHTML"!==i&&"cssText"!==i&&"itemprop"!==i||(n=i)}if(!n||!e[n])return!1;var l=e[n].toLowerCase();return s[n]||(s[n]={}),r[n]||(r[n]={}),!s[n][l]&&(r[n][l]=!0,!0)})).reverse().forEach((function(t){return e.push(t)}));for(var o=Object.keys(r),a=0;a<o.length;a+=1){var i=o[a],c=u({},s[i],r[i]);s[i]=c}return e}),[]).reverse()},I=function(e,t){if(Array.isArray(e)&&e.length)for(var n=0;n<e.length;n+=1)if(e[n][t])return!0;return!1},C=function(e){return Array.isArray(e)?e.join(""):e},Q=function(e,t){return Array.isArray(e)?e.reduce((function(e,n){return function(e,t){for(var n=Object.keys(e),s=0;s<n.length;s+=1)if(t[n[s]]&&t[n[s]].includes(e[n[s]]))return!0;return!1}(n,t)?e.priority.push(n):e.default.push(n),e}),{priority:[],default:[]}):{default:e}},R=function(e,t){var n;return u({},e,((n={})[t]=void 0,n))},D=[f.NOSCRIPT,f.SCRIPT,f.STYLE],T=function(e,t){return void 0===t&&(t=!0),!1===t?String(e):String(e).replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")},O=function(e){return Object.keys(e).reduce((function(t,n){var s=void 0!==e[n]?n+'="'+e[n]+'"':""+n;return t?t+" "+s:s}),"")},P=function(e,t){return void 0===t&&(t={}),Object.keys(e).reduce((function(t,n){return t[A[n]||n]=e[n],t}),t)},F=function(e,t){return t.map((function(t,n){var r,o=((r={key:n})["data-rh"]=!0,r);return Object.keys(t).forEach((function(e){var n=A[e]||e;"innerHTML"===n||"cssText"===n?o.dangerouslySetInnerHTML={__html:t.innerHTML||t.cssText}:o[n]=t[e]})),s.createElement(e,o)}))},N=function(e,t,n){switch(e){case f.TITLE:return{toComponent:function(){return n=t.titleAttributes,(r={key:e=t.title})["data-rh"]=!0,o=P(n,r),[s.createElement(f.TITLE,o,e)];var e,n,r,o},toString:function(){return function(e,t,n,s){var r=O(n),o=C(t);return r?"<"+e+' data-rh="true" '+r+">"+T(o,s)+"</"+e+">":"<"+e+' data-rh="true">'+T(o,s)+"</"+e+">"}(e,t.title,t.titleAttributes,n)}};case"bodyAttributes":case"htmlAttributes":return{toComponent:function(){return P(t)},toString:function(){return O(t)}};default:return{toComponent:function(){return F(e,t)},toString:function(){return function(e,t,n){return t.reduce((function(t,s){var r=Object.keys(s).filter((function(e){return!("innerHTML"===e||"cssText"===e)})).reduce((function(e,t){var r=void 0===s[t]?t:t+'="'+T(s[t],n)+'"';return e?e+" "+r:r}),""),o=s.innerHTML||s.cssText||"",a=-1===D.indexOf(e);return t+"<"+e+' data-rh="true" '+r+(a?"/>":">"+o+"</"+e+">")}),"")}(e,t,n)}}}},B=function(e){var t=e.baseTag,n=e.bodyAttributes,s=e.encode,r=e.htmlAttributes,o=e.noscriptTags,a=e.styleTags,i=e.title,c=void 0===i?"":i,l=e.titleAttributes,d=e.linkTags,p=e.metaTags,u=e.scriptTags,m={toComponent:function(){},toString:function(){return""}};if(e.prioritizeSeoTags){var h=function(e){var t=e.linkTags,n=e.scriptTags,s=e.encode,r=Q(e.metaTags,x),o=Q(t,b),a=Q(n,y);return{priorityMethods:{toComponent:function(){return[].concat(F(f.META,r.priority),F(f.LINK,o.priority),F(f.SCRIPT,a.priority))},toString:function(){return N(f.META,r.priority,s)+" "+N(f.LINK,o.priority,s)+" "+N(f.SCRIPT,a.priority,s)}},metaTags:r.default,linkTags:o.default,scriptTags:a.default}}(e);m=h.priorityMethods,d=h.linkTags,p=h.metaTags,u=h.scriptTags}return{priority:m,base:N(f.BASE,t,s),bodyAttributes:N("bodyAttributes",n,s),htmlAttributes:N("htmlAttributes",r,s),link:N(f.LINK,d,s),meta:N(f.META,p,s),noscript:N(f.NOSCRIPT,o,s),script:N(f.SCRIPT,u,s),style:N(f.STYLE,a,s),title:N(f.TITLE,{title:c,titleAttributes:l},s)}},V=[],G=function(e,t){var n=this;void 0===t&&(t="undefined"!=typeof document),this.instances=[],this.value={setHelmet:function(e){n.context.helmet=e},helmetInstances:{get:function(){return n.canUseDOM?V:n.instances},add:function(e){(n.canUseDOM?V:n.instances).push(e)},remove:function(e){var t=(n.canUseDOM?V:n.instances).indexOf(e);(n.canUseDOM?V:n.instances).splice(t,1)}}},this.context=e,this.canUseDOM=t,t||(e.helmet=B({baseTag:[],bodyAttributes:{},encodeSpecialCharacters:!0,htmlAttributes:{},linkTags:[],metaTags:[],noscriptTags:[],scriptTags:[],styleTags:[],title:"",titleAttributes:{}}))},W=s.createContext({}),U=o().shape({setHelmet:o().func,helmetInstances:o().shape({get:o().func,add:o().func,remove:o().func})}),z="undefined"!=typeof document,H=function(e){function t(n){var s;return(s=e.call(this,n)||this).helmetData=new G(s.props.context,t.canUseDOM),s}return m(t,e),t.prototype.render=function(){return s.createElement(W.Provider,{value:this.helmetData.value},this.props.children)},t}(s.Component);H.canUseDOM=z,H.propTypes={context:o().shape({helmet:o().shape()}),children:o().node.isRequired},H.defaultProps={context:{}},H.displayName="HelmetProvider";var j=function(e,t){var n,s=document.head||document.querySelector(f.HEAD),r=s.querySelectorAll(e+"[data-rh]"),o=[].slice.call(r),a=[];return t&&t.length&&t.forEach((function(t){var s=document.createElement(e);for(var r in t)Object.prototype.hasOwnProperty.call(t,r)&&("innerHTML"===r?s.innerHTML=t.innerHTML:"cssText"===r?s.styleSheet?s.styleSheet.cssText=t.cssText:s.appendChild(document.createTextNode(t.cssText)):s.setAttribute(r,void 0===t[r]?"":t[r]));s.setAttribute("data-rh","true"),o.some((function(e,t){return n=t,s.isEqualNode(e)}))?o.splice(n,1):a.push(s)})),o.forEach((function(e){return e.parentNode.removeChild(e)})),a.forEach((function(e){return s.appendChild(e)})),{oldTags:o,newTags:a}},$=function(e,t){var n=document.getElementsByTagName(e)[0];if(n){for(var s=n.getAttribute("data-rh"),r=s?s.split(","):[],o=[].concat(r),a=Object.keys(t),i=0;i<a.length;i+=1){var c=a[i],l=t[c]||"";n.getAttribute(c)!==l&&n.setAttribute(c,l),-1===r.indexOf(c)&&r.push(c);var d=o.indexOf(c);-1!==d&&o.splice(d,1)}for(var p=o.length-1;p>=0;p-=1)n.removeAttribute(o[p]);r.length===o.length?n.removeAttribute("data-rh"):n.getAttribute("data-rh")!==a.join(",")&&n.setAttribute("data-rh",a.join(","))}},Z=function(e,t){var n=e.baseTag,s=e.htmlAttributes,r=e.linkTags,o=e.metaTags,a=e.noscriptTags,i=e.onChangeClientState,c=e.scriptTags,l=e.styleTags,d=e.title,p=e.titleAttributes;$(f.BODY,e.bodyAttributes),$(f.HTML,s),function(e,t){void 0!==e&&document.title!==e&&(document.title=C(e)),$(f.TITLE,t)}(d,p);var u={baseTag:j(f.BASE,n),linkTags:j(f.LINK,r),metaTags:j(f.META,o),noscriptTags:j(f.NOSCRIPT,a),scriptTags:j(f.SCRIPT,c),styleTags:j(f.STYLE,l)},m={},h={};Object.keys(u).forEach((function(e){var t=u[e],n=t.newTags,s=t.oldTags;n.length&&(m[e]=n),s.length&&(h[e]=u[e].oldTags)})),t&&t(),i(e,m,h)},K=null,X=function(e){function t(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).rendered=!1,t}m(t,e);var n=t.prototype;return n.shouldComponentUpdate=function(e){return!p()(e,this.props)},n.componentDidUpdate=function(){this.emitChange()},n.componentWillUnmount=function(){this.props.context.helmetInstances.remove(this),this.emitChange()},n.emitChange=function(){var e,t,n=this.props.context,s=n.setHelmet,r=null,o=(e=n.helmetInstances.get().map((function(e){var t=u({},e.props);return delete t.context,t})),{baseTag:_(["href"],e),bodyAttributes:k("bodyAttributes",e),defer:E(e,"defer"),encode:E(e,"encodeSpecialCharacters"),htmlAttributes:k("htmlAttributes",e),linkTags:w(f.LINK,["rel","href"],e),metaTags:w(f.META,["name","charset","http-equiv","property","itemprop"],e),noscriptTags:w(f.NOSCRIPT,["innerHTML"],e),onChangeClientState:M(e),scriptTags:w(f.SCRIPT,["src","innerHTML"],e),styleTags:w(f.STYLE,["cssText"],e),title:L(e),titleAttributes:k("titleAttributes",e),prioritizeSeoTags:I(e,"prioritizeSeoTags")});H.canUseDOM?(t=o,K&&cancelAnimationFrame(K),t.defer?K=requestAnimationFrame((function(){Z(t,(function(){K=null}))})):(Z(t),K=null)):B&&(r=B(o)),s(r)},n.init=function(){this.rendered||(this.rendered=!0,this.props.context.helmetInstances.add(this),this.emitChange())},n.render=function(){return this.init(),null},t}(s.Component);X.propTypes={context:U.isRequired},X.displayName="HelmetDispatcher";var Y=["children"],q=["children"],J=function(e){function t(){return e.apply(this,arguments)||this}m(t,e);var n=t.prototype;return n.shouldComponentUpdate=function(e){return!i()(R(this.props,"helmetData"),R(e,"helmetData"))},n.mapNestedChildrenToProps=function(e,t){if(!t)return null;switch(e.type){case f.SCRIPT:case f.NOSCRIPT:return{innerHTML:t};case f.STYLE:return{cssText:t};default:throw new Error("<"+e.type+" /> elements are self-closing and can not contain children. Refer to our API for more information.")}},n.flattenArrayTypeChildren=function(e){var t,n=e.child,s=e.arrayTypeChildren;return u({},s,((t={})[n.type]=[].concat(s[n.type]||[],[u({},e.newChildProps,this.mapNestedChildrenToProps(n,e.nestedChildren))]),t))},n.mapObjectTypeChildren=function(e){var t,n,s=e.child,r=e.newProps,o=e.newChildProps,a=e.nestedChildren;switch(s.type){case f.TITLE:return u({},r,((t={})[s.type]=a,t.titleAttributes=u({},o),t));case f.BODY:return u({},r,{bodyAttributes:u({},o)});case f.HTML:return u({},r,{htmlAttributes:u({},o)});default:return u({},r,((n={})[s.type]=u({},o),n))}},n.mapArrayTypeChildrenToProps=function(e,t){var n=u({},t);return Object.keys(e).forEach((function(t){var s;n=u({},n,((s={})[t]=e[t],s))})),n},n.warnOnInvalidChildren=function(e,t){return l()(v.some((function(t){return e.type===t})),"function"==typeof e.type?"You may be attempting to nest <Helmet> components within each other, which is not allowed. Refer to our API for more information.":"Only elements types "+v.join(", ")+" are allowed. Helmet does not support rendering <"+e.type+"> elements. Refer to our API for more information."),l()(!t||"string"==typeof t||Array.isArray(t)&&!t.some((function(e){return"string"!=typeof e})),"Helmet expects a string as a child of <"+e.type+">. Did you forget to wrap your children in braces? ( <"+e.type+">{``}</"+e.type+"> ) Refer to our API for more information."),!0},n.mapChildrenToProps=function(e,t){var n=this,r={};return s.Children.forEach(e,(function(e){if(e&&e.props){var s=e.props,o=s.children,a=g(s,Y),i=Object.keys(a).reduce((function(e,t){return e[S[t]||t]=a[t],e}),{}),c=e.type;switch("symbol"==typeof c?c=c.toString():n.warnOnInvalidChildren(e,o),c){case f.FRAGMENT:t=n.mapChildrenToProps(o,t);break;case f.LINK:case f.META:case f.NOSCRIPT:case f.SCRIPT:case f.STYLE:r=n.flattenArrayTypeChildren({child:e,arrayTypeChildren:r,newChildProps:i,nestedChildren:o});break;default:t=n.mapObjectTypeChildren({child:e,newProps:t,newChildProps:i,nestedChildren:o})}}})),this.mapArrayTypeChildrenToProps(r,t)},n.render=function(){var e=this.props,t=e.children,n=g(e,q),r=u({},n),o=n.helmetData;return t&&(r=this.mapChildrenToProps(t,r)),!o||o instanceof G||(o=new G(o.context,o.instances)),o?s.createElement(X,u({},r,{context:o.value,helmetData:void 0})):s.createElement(W.Consumer,null,(function(e){return s.createElement(X,u({},r,{context:e}))}))},t}(s.Component);J.propTypes={base:o().object,bodyAttributes:o().object,children:o().oneOfType([o().arrayOf(o().node),o().node]),defaultTitle:o().string,defer:o().bool,encodeSpecialCharacters:o().bool,htmlAttributes:o().object,link:o().arrayOf(o().object),meta:o().arrayOf(o().object),noscript:o().arrayOf(o().object),onChangeClientState:o().func,script:o().arrayOf(o().object),style:o().arrayOf(o().object),title:o().string,titleAttributes:o().object,titleTemplate:o().string,prioritizeSeoTags:o().bool,helmetData:o().object},J.defaultProps={defer:!0,encodeSpecialCharacters:!0,prioritizeSeoTags:!1},J.displayName="Helmet"},69921:(e,t)=>{"use strict";var n="function"==typeof Symbol&&Symbol.for,s=n?Symbol.for("react.element"):60103,r=n?Symbol.for("react.portal"):60106,o=n?Symbol.for("react.fragment"):60107,a=n?Symbol.for("react.strict_mode"):60108,i=n?Symbol.for("react.profiler"):60114,c=n?Symbol.for("react.provider"):60109,l=n?Symbol.for("react.context"):60110,d=n?Symbol.for("react.async_mode"):60111,p=n?Symbol.for("react.concurrent_mode"):60111,u=n?Symbol.for("react.forward_ref"):60112,m=n?Symbol.for("react.suspense"):60113,h=n?Symbol.for("react.suspense_list"):60120,g=n?Symbol.for("react.memo"):60115,f=n?Symbol.for("react.lazy"):60116,b=n?Symbol.for("react.block"):60121,y=n?Symbol.for("react.fundamental"):60117,x=n?Symbol.for("react.responder"):60118,v=n?Symbol.for("react.scope"):60119;function A(e){if("object"==typeof e&&null!==e){var t=e.$$typeof;switch(t){case s:switch(e=e.type){case d:case p:case o:case i:case a:case m:return e;default:switch(e=e&&e.$$typeof){case l:case u:case f:case g:case c:return e;default:return t}}case r:return t}}}function S(e){return A(e)===p}t.AsyncMode=d,t.ConcurrentMode=p,t.ContextConsumer=l,t.ContextProvider=c,t.Element=s,t.ForwardRef=u,t.Fragment=o,t.Lazy=f,t.Memo=g,t.Portal=r,t.Profiler=i,t.StrictMode=a,t.Suspense=m,t.isAsyncMode=function(e){return S(e)||A(e)===d},t.isConcurrentMode=S,t.isContextConsumer=function(e){return A(e)===l},t.isContextProvider=function(e){return A(e)===c},t.isElement=function(e){return"object"==typeof e&&null!==e&&e.$$typeof===s},t.isForwardRef=function(e){return A(e)===u},t.isFragment=function(e){return A(e)===o},t.isLazy=function(e){return A(e)===f},t.isMemo=function(e){return A(e)===g},t.isPortal=function(e){return A(e)===r},t.isProfiler=function(e){return A(e)===i},t.isStrictMode=function(e){return A(e)===a},t.isSuspense=function(e){return A(e)===m},t.isValidElementType=function(e){return"string"==typeof e||"function"==typeof e||e===o||e===p||e===i||e===a||e===m||e===h||"object"==typeof e&&null!==e&&(e.$$typeof===f||e.$$typeof===g||e.$$typeof===c||e.$$typeof===l||e.$$typeof===u||e.$$typeof===y||e.$$typeof===x||e.$$typeof===v||e.$$typeof===b)},t.typeOf=A},59864:(e,t,n)=>{"use strict";e.exports=n(69921)},68356:(e,t,n)=>{"use strict";function s(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,e.__proto__=t}function r(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(){return a=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},a.apply(this,arguments)}var i=n(67294),c=n(45697),l=[],d=[];function p(e){var t=e(),n={loading:!0,loaded:null,error:null};return n.promise=t.then((function(e){return n.loading=!1,n.loaded=e,e})).catch((function(e){throw n.loading=!1,n.error=e,e})),n}function u(e){var t={loading:!1,loaded:{},error:null},n=[];try{Object.keys(e).forEach((function(s){var r=p(e[s]);r.loading?t.loading=!0:(t.loaded[s]=r.loaded,t.error=r.error),n.push(r.promise),r.promise.then((function(e){t.loaded[s]=e})).catch((function(e){t.error=e}))}))}catch(s){t.error=s}return t.promise=Promise.all(n).then((function(e){return t.loading=!1,e})).catch((function(e){throw t.loading=!1,e})),t}function m(e,t){return i.createElement((n=e)&&n.__esModule?n.default:n,t);var n}function h(e,t){var p,u;if(!t.loading)throw new Error("react-loadable requires a `loading` component");var h=a({loader:null,loading:null,delay:200,timeout:null,render:m,webpack:null,modules:null},t),g=null;function f(){return g||(g=e(h.loader)),g.promise}return l.push(f),"function"==typeof h.webpack&&d.push((function(){if((0,h.webpack)().every((function(e){return void 0!==e&&void 0!==n.m[e]})))return f()})),u=p=function(t){function n(n){var s;return o(r(r(s=t.call(this,n)||this)),"retry",(function(){s.setState({error:null,loading:!0,timedOut:!1}),g=e(h.loader),s._loadModule()})),f(),s.state={error:g.error,pastDelay:!1,timedOut:!1,loading:g.loading,loaded:g.loaded},s}s(n,t),n.preload=function(){return f()};var a=n.prototype;return a.UNSAFE_componentWillMount=function(){this._loadModule()},a.componentDidMount=function(){this._mounted=!0},a._loadModule=function(){var e=this;if(this.context.loadable&&Array.isArray(h.modules)&&h.modules.forEach((function(t){e.context.loadable.report(t)})),g.loading){var t=function(t){e._mounted&&e.setState(t)};"number"==typeof h.delay&&(0===h.delay?this.setState({pastDelay:!0}):this._delay=setTimeout((function(){t({pastDelay:!0})}),h.delay)),"number"==typeof h.timeout&&(this._timeout=setTimeout((function(){t({timedOut:!0})}),h.timeout));var n=function(){t({error:g.error,loaded:g.loaded,loading:g.loading}),e._clearTimeouts()};g.promise.then((function(){return n(),null})).catch((function(e){return n(),null}))}},a.componentWillUnmount=function(){this._mounted=!1,this._clearTimeouts()},a._clearTimeouts=function(){clearTimeout(this._delay),clearTimeout(this._timeout)},a.render=function(){return this.state.loading||this.state.error?i.createElement(h.loading,{isLoading:this.state.loading,pastDelay:this.state.pastDelay,timedOut:this.state.timedOut,error:this.state.error,retry:this.retry}):this.state.loaded?h.render(this.state.loaded,this.props):null},n}(i.Component),o(p,"contextTypes",{loadable:c.shape({report:c.func.isRequired})}),u}function g(e){return h(p,e)}g.Map=function(e){if("function"!=typeof e.render)throw new Error("LoadableMap requires a `render(loaded, props)` function");return h(u,e)};var f=function(e){function t(){return e.apply(this,arguments)||this}s(t,e);var n=t.prototype;return n.getChildContext=function(){return{loadable:{report:this.props.report}}},n.render=function(){return i.Children.only(this.props.children)},t}(i.Component);function b(e){for(var t=[];e.length;){var n=e.pop();t.push(n())}return Promise.all(t).then((function(){if(e.length)return b(e)}))}o(f,"propTypes",{report:c.func.isRequired}),o(f,"childContextTypes",{loadable:c.shape({report:c.func.isRequired}).isRequired}),g.Capture=f,g.preloadAll=function(){return new Promise((function(e,t){b(l).then(e,t)}))},g.preloadReady=function(){return new Promise((function(e,t){b(d).then(e,e)}))},e.exports=g},18790:(e,t,n)=>{"use strict";n.d(t,{H:()=>i,f:()=>a});var s=n(16550),r=n(83117),o=n(67294);function a(e,t,n){return void 0===n&&(n=[]),e.some((function(e){var r=e.path?(0,s.LX)(t,e):n.length?n[n.length-1].match:s.F0.computeRootMatch(t);return r&&(n.push({route:e,match:r}),e.routes&&a(e.routes,t,n)),r})),n}function i(e,t,n){return void 0===t&&(t={}),void 0===n&&(n={}),e?o.createElement(s.rs,n,e.map((function(e,n){return o.createElement(s.AW,{key:e.key||n,path:e.path,exact:e.exact,strict:e.strict,render:function(n){return e.render?e.render((0,r.Z)({},n,{},t,{route:e})):o.createElement(e.component,(0,r.Z)({},n,t,{route:e}))}})}))):null}},73727:(e,t,n)=>{"use strict";n.d(t,{OL:()=>x,VK:()=>d,rU:()=>f});var s=n(16550),r=n(90144),o=n(67294),a=n(99318),i=n(83117),c=n(80102),l=n(2177),d=function(e){function t(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).history=(0,a.lX)(t.props),t}return(0,r.Z)(t,e),t.prototype.render=function(){return o.createElement(s.F0,{history:this.history,children:this.props.children})},t}(o.Component);o.Component;var p=function(e,t){return"function"==typeof e?e(t):e},u=function(e,t){return"string"==typeof e?(0,a.ob)(e,null,null,t):e},m=function(e){return e},h=o.forwardRef;void 0===h&&(h=m);var g=h((function(e,t){var n=e.innerRef,s=e.navigate,r=e.onClick,a=(0,c.Z)(e,["innerRef","navigate","onClick"]),l=a.target,d=(0,i.Z)({},a,{onClick:function(e){try{r&&r(e)}catch(t){throw e.preventDefault(),t}e.defaultPrevented||0!==e.button||l&&"_self"!==l||function(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}(e)||(e.preventDefault(),s())}});return d.ref=m!==h&&t||n,o.createElement("a",d)}));var f=h((function(e,t){var n=e.component,r=void 0===n?g:n,d=e.replace,f=e.to,b=e.innerRef,y=(0,c.Z)(e,["component","replace","to","innerRef"]);return o.createElement(s.s6.Consumer,null,(function(e){e||(0,l.Z)(!1);var n=e.history,s=u(p(f,e.location),e.location),c=s?n.createHref(s):"",g=(0,i.Z)({},y,{href:c,navigate:function(){var t=p(f,e.location),s=(0,a.Ep)(e.location)===(0,a.Ep)(u(t));(d||s?n.replace:n.push)(t)}});return m!==h?g.ref=t||b:g.innerRef=b,o.createElement(r,g)}))})),b=function(e){return e},y=o.forwardRef;void 0===y&&(y=b);var x=y((function(e,t){var n=e["aria-current"],r=void 0===n?"page":n,a=e.activeClassName,d=void 0===a?"active":a,m=e.activeStyle,h=e.className,g=e.exact,x=e.isActive,v=e.location,A=e.sensitive,S=e.strict,E=e.style,L=e.to,M=e.innerRef,k=(0,c.Z)(e,["aria-current","activeClassName","activeStyle","className","exact","isActive","location","sensitive","strict","style","to","innerRef"]);return o.createElement(s.s6.Consumer,null,(function(e){e||(0,l.Z)(!1);var n=v||e.location,a=u(p(L,n),n),c=a.pathname,_=c&&c.replace(/([.+*?=^!:${}()[\]|/\\])/g,"\\$1"),w=_?(0,s.LX)(n.pathname,{path:_,exact:g,sensitive:A,strict:S}):null,I=!!(x?x(w,n):w),C="function"==typeof h?h(I):h,Q="function"==typeof E?E(I):E;I&&(C=function(){for(var e=arguments.length,t=new Array(e),n=0;n<e;n++)t[n]=arguments[n];return t.filter((function(e){return e})).join(" ")}(C,d),Q=(0,i.Z)({},Q,m));var R=(0,i.Z)({"aria-current":I&&r||null,className:C,style:Q,to:a},k);return b!==y?R.ref=t||M:R.innerRef=M,o.createElement(f,R)}))}))},16550:(e,t,n)=>{"use strict";n.d(t,{AW:()=>L,F0:()=>v,LX:()=>E,TH:()=>D,k6:()=>R,rs:()=>C,s6:()=>x});var s=n(90144),r=n(67294),o=n(45697),a=n.n(o),i=n(99318),c=n(2177),l=n(83117),d=n(39658),p=n.n(d),u=(n(59864),n(80102)),m=(n(8679),1073741823),h="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:void 0!==n.g?n.g:{};function g(e){var t=[];return{on:function(e){t.push(e)},off:function(e){t=t.filter((function(t){return t!==e}))},get:function(){return e},set:function(n,s){e=n,t.forEach((function(t){return t(e,s)}))}}}var f=r.createContext||function(e,t){var n,o,i="__create-react-context-"+function(){var e="__global_unique_id__";return h[e]=(h[e]||0)+1}()+"__",c=function(e){function n(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).emitter=g(t.props.value),t}(0,s.Z)(n,e);var r=n.prototype;return r.getChildContext=function(){var e;return(e={})[i]=this.emitter,e},r.componentWillReceiveProps=function(e){if(this.props.value!==e.value){var n,s=this.props.value,r=e.value;((o=s)===(a=r)?0!==o||1/o==1/a:o!=o&&a!=a)?n=0:(n="function"==typeof t?t(s,r):m,0!==(n|=0)&&this.emitter.set(e.value,n))}var o,a},r.render=function(){return this.props.children},n}(r.Component);c.childContextTypes=((n={})[i]=a().object.isRequired,n);var l=function(t){function n(){for(var e,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(e=t.call.apply(t,[this].concat(s))||this).observedBits=void 0,e.state={value:e.getValue()},e.onUpdate=function(t,n){0!=((0|e.observedBits)&n)&&e.setState({value:e.getValue()})},e}(0,s.Z)(n,t);var r=n.prototype;return r.componentWillReceiveProps=function(e){var t=e.observedBits;this.observedBits=null==t?m:t},r.componentDidMount=function(){this.context[i]&&this.context[i].on(this.onUpdate);var e=this.props.observedBits;this.observedBits=null==e?m:e},r.componentWillUnmount=function(){this.context[i]&&this.context[i].off(this.onUpdate)},r.getValue=function(){return this.context[i]?this.context[i].get():e},r.render=function(){return(e=this.props.children,Array.isArray(e)?e[0]:e)(this.state.value);var e},n}(r.Component);return l.contextTypes=((o={})[i]=a().object,o),{Provider:c,Consumer:l}},b=function(e){var t=f();return t.displayName=e,t},y=b("Router-History"),x=b("Router"),v=function(e){function t(t){var n;return(n=e.call(this,t)||this).state={location:t.history.location},n._isMounted=!1,n._pendingLocation=null,t.staticContext||(n.unlisten=t.history.listen((function(e){n._pendingLocation=e}))),n}(0,s.Z)(t,e),t.computeRootMatch=function(e){return{path:"/",url:"/",params:{},isExact:"/"===e}};var n=t.prototype;return n.componentDidMount=function(){var e=this;this._isMounted=!0,this.unlisten&&this.unlisten(),this.props.staticContext||(this.unlisten=this.props.history.listen((function(t){e._isMounted&&e.setState({location:t})}))),this._pendingLocation&&this.setState({location:this._pendingLocation})},n.componentWillUnmount=function(){this.unlisten&&(this.unlisten(),this._isMounted=!1,this._pendingLocation=null)},n.render=function(){return r.createElement(x.Provider,{value:{history:this.props.history,location:this.state.location,match:t.computeRootMatch(this.state.location.pathname),staticContext:this.props.staticContext}},r.createElement(y.Provider,{children:this.props.children||null,value:this.props.history}))},t}(r.Component);r.Component;r.Component;var A={},S=0;function E(e,t){void 0===t&&(t={}),("string"==typeof t||Array.isArray(t))&&(t={path:t});var n=t,s=n.path,r=n.exact,o=void 0!==r&&r,a=n.strict,i=void 0!==a&&a,c=n.sensitive,l=void 0!==c&&c;return[].concat(s).reduce((function(t,n){if(!n&&""!==n)return null;if(t)return t;var s=function(e,t){var n=""+t.end+t.strict+t.sensitive,s=A[n]||(A[n]={});if(s[e])return s[e];var r=[],o={regexp:p()(e,r,t),keys:r};return S<1e4&&(s[e]=o,S++),o}(n,{end:o,strict:i,sensitive:l}),r=s.regexp,a=s.keys,c=r.exec(e);if(!c)return null;var d=c[0],u=c.slice(1),m=e===d;return o&&!m?null:{path:n,url:"/"===n&&""===d?"/":d,isExact:m,params:a.reduce((function(e,t,n){return e[t.name]=u[n],e}),{})}}),null)}var L=function(e){function t(){return e.apply(this,arguments)||this}return(0,s.Z)(t,e),t.prototype.render=function(){var e=this;return r.createElement(x.Consumer,null,(function(t){t||(0,c.Z)(!1);var n=e.props.location||t.location,s=e.props.computedMatch?e.props.computedMatch:e.props.path?E(n.pathname,e.props):t.match,o=(0,l.Z)({},t,{location:n,match:s}),a=e.props,i=a.children,d=a.component,p=a.render;return Array.isArray(i)&&function(e){return 0===r.Children.count(e)}(i)&&(i=null),r.createElement(x.Provider,{value:o},o.match?i?"function"==typeof i?i(o):i:d?r.createElement(d,o):p?p(o):null:"function"==typeof i?i(o):null)}))},t}(r.Component);function M(e){return"/"===e.charAt(0)?e:"/"+e}function k(e,t){if(!e)return t;var n=M(e);return 0!==t.pathname.indexOf(n)?t:(0,l.Z)({},t,{pathname:t.pathname.substr(n.length)})}function _(e){return"string"==typeof e?e:(0,i.Ep)(e)}function w(e){return function(){(0,c.Z)(!1)}}function I(){}r.Component;var C=function(e){function t(){return e.apply(this,arguments)||this}return(0,s.Z)(t,e),t.prototype.render=function(){var e=this;return r.createElement(x.Consumer,null,(function(t){t||(0,c.Z)(!1);var n,s,o=e.props.location||t.location;return r.Children.forEach(e.props.children,(function(e){if(null==s&&r.isValidElement(e)){n=e;var a=e.props.path||e.props.from;s=a?E(o.pathname,(0,l.Z)({},e.props,{path:a})):t.match}})),s?r.cloneElement(n,{location:o,computedMatch:s}):null}))},t}(r.Component);var Q=r.useContext;function R(){return Q(y)}function D(){return Q(x).location}},39658:(e,t,n)=>{var s=n(5826);e.exports=m,e.exports.parse=o,e.exports.compile=function(e,t){return i(o(e,t),t)},e.exports.tokensToFunction=i,e.exports.tokensToRegExp=u;var r=new RegExp(["(\\\\.)","([\\/.])?(?:(?:\\:(\\w+)(?:\\(((?:\\\\.|[^\\\\()])+)\\))?|\\(((?:\\\\.|[^\\\\()])+)\\))([+*?])?|(\\*))"].join("|"),"g");function o(e,t){for(var n,s=[],o=0,a=0,i="",d=t&&t.delimiter||"/";null!=(n=r.exec(e));){var p=n[0],u=n[1],m=n.index;if(i+=e.slice(a,m),a=m+p.length,u)i+=u[1];else{var h=e[a],g=n[2],f=n[3],b=n[4],y=n[5],x=n[6],v=n[7];i&&(s.push(i),i="");var A=null!=g&&null!=h&&h!==g,S="+"===x||"*"===x,E="?"===x||"*"===x,L=n[2]||d,M=b||y;s.push({name:f||o++,prefix:g||"",delimiter:L,optional:E,repeat:S,partial:A,asterisk:!!v,pattern:M?l(M):v?".*":"[^"+c(L)+"]+?"})}}return a<e.length&&(i+=e.substr(a)),i&&s.push(i),s}function a(e){return encodeURI(e).replace(/[\/?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()}))}function i(e,t){for(var n=new Array(e.length),r=0;r<e.length;r++)"object"==typeof e[r]&&(n[r]=new RegExp("^(?:"+e[r].pattern+")$",p(t)));return function(t,r){for(var o="",i=t||{},c=(r||{}).pretty?a:encodeURIComponent,l=0;l<e.length;l++){var d=e[l];if("string"!=typeof d){var p,u=i[d.name];if(null==u){if(d.optional){d.partial&&(o+=d.prefix);continue}throw new TypeError('Expected "'+d.name+'" to be defined')}if(s(u)){if(!d.repeat)throw new TypeError('Expected "'+d.name+'" to not repeat, but received `'+JSON.stringify(u)+"`");if(0===u.length){if(d.optional)continue;throw new TypeError('Expected "'+d.name+'" to not be empty')}for(var m=0;m<u.length;m++){if(p=c(u[m]),!n[l].test(p))throw new TypeError('Expected all "'+d.name+'" to match "'+d.pattern+'", but received `'+JSON.stringify(p)+"`");o+=(0===m?d.prefix:d.delimiter)+p}}else{if(p=d.asterisk?encodeURI(u).replace(/[?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()})):c(u),!n[l].test(p))throw new TypeError('Expected "'+d.name+'" to match "'+d.pattern+'", but received "'+p+'"');o+=d.prefix+p}}else o+=d}return o}}function c(e){return e.replace(/([.+*?=^!:${}()[\]|\/\\])/g,"\\$1")}function l(e){return e.replace(/([=!:$\/()])/g,"\\$1")}function d(e,t){return e.keys=t,e}function p(e){return e&&e.sensitive?"":"i"}function u(e,t,n){s(t)||(n=t||n,t=[]);for(var r=(n=n||{}).strict,o=!1!==n.end,a="",i=0;i<e.length;i++){var l=e[i];if("string"==typeof l)a+=c(l);else{var u=c(l.prefix),m="(?:"+l.pattern+")";t.push(l),l.repeat&&(m+="(?:"+u+m+")*"),a+=m=l.optional?l.partial?u+"("+m+")?":"(?:"+u+"("+m+"))?":u+"("+m+")"}}var h=c(n.delimiter||"/"),g=a.slice(-h.length)===h;return r||(a=(g?a.slice(0,-h.length):a)+"(?:"+h+"(?=$))?"),a+=o?"$":r&&g?"":"(?="+h+"|$)",d(new RegExp("^"+a,p(n)),t)}function m(e,t,n){return s(t)||(n=t||n,t=[]),n=n||{},e instanceof RegExp?function(e,t){var n=e.source.match(/\((?!\?)/g);if(n)for(var s=0;s<n.length;s++)t.push({name:s,prefix:null,delimiter:null,optional:!1,repeat:!1,partial:!1,asterisk:!1,pattern:null});return d(e,t)}(e,t):s(e)?function(e,t,n){for(var s=[],r=0;r<e.length;r++)s.push(m(e[r],t,n).source);return d(new RegExp("(?:"+s.join("|")+")",p(n)),t)}(e,t,n):function(e,t,n){return u(o(e,n),t,n)}(e,t,n)}},72408:(e,t,n)=>{"use strict";var s=n(27418),r="function"==typeof Symbol&&Symbol.for,o=r?Symbol.for("react.element"):60103,a=r?Symbol.for("react.portal"):60106,i=r?Symbol.for("react.fragment"):60107,c=r?Symbol.for("react.strict_mode"):60108,l=r?Symbol.for("react.profiler"):60114,d=r?Symbol.for("react.provider"):60109,p=r?Symbol.for("react.context"):60110,u=r?Symbol.for("react.forward_ref"):60112,m=r?Symbol.for("react.suspense"):60113,h=r?Symbol.for("react.memo"):60115,g=r?Symbol.for("react.lazy"):60116,f="function"==typeof Symbol&&Symbol.iterator;function b(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n<arguments.length;n++)t+="&args[]="+encodeURIComponent(arguments[n]);return"Minified React error #"+e+"; visit "+t+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}var y={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},x={};function v(e,t,n){this.props=e,this.context=t,this.refs=x,this.updater=n||y}function A(){}function S(e,t,n){this.props=e,this.context=t,this.refs=x,this.updater=n||y}v.prototype.isReactComponent={},v.prototype.setState=function(e,t){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error(b(85));this.updater.enqueueSetState(this,e,t,"setState")},v.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},A.prototype=v.prototype;var E=S.prototype=new A;E.constructor=S,s(E,v.prototype),E.isPureReactComponent=!0;var L={current:null},M=Object.prototype.hasOwnProperty,k={key:!0,ref:!0,__self:!0,__source:!0};function _(e,t,n){var s,r={},a=null,i=null;if(null!=t)for(s in void 0!==t.ref&&(i=t.ref),void 0!==t.key&&(a=""+t.key),t)M.call(t,s)&&!k.hasOwnProperty(s)&&(r[s]=t[s]);var c=arguments.length-2;if(1===c)r.children=n;else if(1<c){for(var l=Array(c),d=0;d<c;d++)l[d]=arguments[d+2];r.children=l}if(e&&e.defaultProps)for(s in c=e.defaultProps)void 0===r[s]&&(r[s]=c[s]);return{$$typeof:o,type:e,key:a,ref:i,props:r,_owner:L.current}}function w(e){return"object"==typeof e&&null!==e&&e.$$typeof===o}var I=/\/+/g,C=[];function Q(e,t,n,s){if(C.length){var r=C.pop();return r.result=e,r.keyPrefix=t,r.func=n,r.context=s,r.count=0,r}return{result:e,keyPrefix:t,func:n,context:s,count:0}}function R(e){e.result=null,e.keyPrefix=null,e.func=null,e.context=null,e.count=0,10>C.length&&C.push(e)}function D(e,t,n,s){var r=typeof e;"undefined"!==r&&"boolean"!==r||(e=null);var i=!1;if(null===e)i=!0;else switch(r){case"string":case"number":i=!0;break;case"object":switch(e.$$typeof){case o:case a:i=!0}}if(i)return n(s,e,""===t?"."+O(e,0):t),1;if(i=0,t=""===t?".":t+":",Array.isArray(e))for(var c=0;c<e.length;c++){var l=t+O(r=e[c],c);i+=D(r,l,n,s)}else if(null===e||"object"!=typeof e?l=null:l="function"==typeof(l=f&&e[f]||e["@@iterator"])?l:null,"function"==typeof l)for(e=l.call(e),c=0;!(r=e.next()).done;)i+=D(r=r.value,l=t+O(r,c++),n,s);else if("object"===r)throw n=""+e,Error(b(31,"[object Object]"===n?"object with keys {"+Object.keys(e).join(", ")+"}":n,""));return i}function T(e,t,n){return null==e?0:D(e,"",t,n)}function O(e,t){return"object"==typeof e&&null!==e&&null!=e.key?function(e){var t={"=":"=0",":":"=2"};return"$"+(""+e).replace(/[=:]/g,(function(e){return t[e]}))}(e.key):t.toString(36)}function P(e,t){e.func.call(e.context,t,e.count++)}function F(e,t,n){var s=e.result,r=e.keyPrefix;e=e.func.call(e.context,t,e.count++),Array.isArray(e)?N(e,s,n,(function(e){return e})):null!=e&&(w(e)&&(e=function(e,t){return{$$typeof:o,type:e.type,key:t,ref:e.ref,props:e.props,_owner:e._owner}}(e,r+(!e.key||t&&t.key===e.key?"":(""+e.key).replace(I,"$&/")+"/")+n)),s.push(e))}function N(e,t,n,s,r){var o="";null!=n&&(o=(""+n).replace(I,"$&/")+"/"),T(e,F,t=Q(t,o,s,r)),R(t)}var B={current:null};function V(){var e=B.current;if(null===e)throw Error(b(321));return e}var G={ReactCurrentDispatcher:B,ReactCurrentBatchConfig:{suspense:null},ReactCurrentOwner:L,IsSomeRendererActing:{current:!1},assign:s};t.Children={map:function(e,t,n){if(null==e)return e;var s=[];return N(e,s,null,t,n),s},forEach:function(e,t,n){if(null==e)return e;T(e,P,t=Q(null,null,t,n)),R(t)},count:function(e){return T(e,(function(){return null}),null)},toArray:function(e){var t=[];return N(e,t,null,(function(e){return e})),t},only:function(e){if(!w(e))throw Error(b(143));return e}},t.Component=v,t.Fragment=i,t.Profiler=l,t.PureComponent=S,t.StrictMode=c,t.Suspense=m,t.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=G,t.cloneElement=function(e,t,n){if(null==e)throw Error(b(267,e));var r=s({},e.props),a=e.key,i=e.ref,c=e._owner;if(null!=t){if(void 0!==t.ref&&(i=t.ref,c=L.current),void 0!==t.key&&(a=""+t.key),e.type&&e.type.defaultProps)var l=e.type.defaultProps;for(d in t)M.call(t,d)&&!k.hasOwnProperty(d)&&(r[d]=void 0===t[d]&&void 0!==l?l[d]:t[d])}var d=arguments.length-2;if(1===d)r.children=n;else if(1<d){l=Array(d);for(var p=0;p<d;p++)l[p]=arguments[p+2];r.children=l}return{$$typeof:o,type:e.type,key:a,ref:i,props:r,_owner:c}},t.createContext=function(e,t){return void 0===t&&(t=null),(e={$$typeof:p,_calculateChangedBits:t,_currentValue:e,_currentValue2:e,_threadCount:0,Provider:null,Consumer:null}).Provider={$$typeof:d,_context:e},e.Consumer=e},t.createElement=_,t.createFactory=function(e){var t=_.bind(null,e);return t.type=e,t},t.createRef=function(){return{current:null}},t.forwardRef=function(e){return{$$typeof:u,render:e}},t.isValidElement=w,t.lazy=function(e){return{$$typeof:g,_ctor:e,_status:-1,_result:null}},t.memo=function(e,t){return{$$typeof:h,type:e,compare:void 0===t?null:t}},t.useCallback=function(e,t){return V().useCallback(e,t)},t.useContext=function(e,t){return V().useContext(e,t)},t.useDebugValue=function(){},t.useEffect=function(e,t){return V().useEffect(e,t)},t.useImperativeHandle=function(e,t,n){return V().useImperativeHandle(e,t,n)},t.useLayoutEffect=function(e,t){return V().useLayoutEffect(e,t)},t.useMemo=function(e,t){return V().useMemo(e,t)},t.useReducer=function(e,t,n){return V().useReducer(e,t,n)},t.useRef=function(e){return V().useRef(e)},t.useState=function(e){return V().useState(e)},t.version="16.14.0"},67294:(e,t,n)=>{"use strict";e.exports=n(72408)},60053:(e,t)=>{"use strict";var n,s,r,o,a;if("undefined"==typeof window||"function"!=typeof MessageChannel){var i=null,c=null,l=function(){if(null!==i)try{var e=t.unstable_now();i(!0,e),i=null}catch(n){throw setTimeout(l,0),n}},d=Date.now();t.unstable_now=function(){return Date.now()-d},n=function(e){null!==i?setTimeout(n,0,e):(i=e,setTimeout(l,0))},s=function(e,t){c=setTimeout(e,t)},r=function(){clearTimeout(c)},o=function(){return!1},a=t.unstable_forceFrameRate=function(){}}else{var p=window.performance,u=window.Date,m=window.setTimeout,h=window.clearTimeout;if("undefined"!=typeof console){var g=window.cancelAnimationFrame;"function"!=typeof window.requestAnimationFrame&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills"),"function"!=typeof g&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills")}if("object"==typeof p&&"function"==typeof p.now)t.unstable_now=function(){return p.now()};else{var f=u.now();t.unstable_now=function(){return u.now()-f}}var b=!1,y=null,x=-1,v=5,A=0;o=function(){return t.unstable_now()>=A},a=function(){},t.unstable_forceFrameRate=function(e){0>e||125<e?console.error("forceFrameRate takes a positive int between 0 and 125, forcing framerates higher than 125 fps is not unsupported"):v=0<e?Math.floor(1e3/e):5};var S=new MessageChannel,E=S.port2;S.port1.onmessage=function(){if(null!==y){var e=t.unstable_now();A=e+v;try{y(!0,e)?E.postMessage(null):(b=!1,y=null)}catch(n){throw E.postMessage(null),n}}else b=!1},n=function(e){y=e,b||(b=!0,E.postMessage(null))},s=function(e,n){x=m((function(){e(t.unstable_now())}),n)},r=function(){h(x),x=-1}}function L(e,t){var n=e.length;e.push(t);e:for(;;){var s=n-1>>>1,r=e[s];if(!(void 0!==r&&0<_(r,t)))break e;e[s]=t,e[n]=r,n=s}}function M(e){return void 0===(e=e[0])?null:e}function k(e){var t=e[0];if(void 0!==t){var n=e.pop();if(n!==t){e[0]=n;e:for(var s=0,r=e.length;s<r;){var o=2*(s+1)-1,a=e[o],i=o+1,c=e[i];if(void 0!==a&&0>_(a,n))void 0!==c&&0>_(c,a)?(e[s]=c,e[i]=n,s=i):(e[s]=a,e[o]=n,s=o);else{if(!(void 0!==c&&0>_(c,n)))break e;e[s]=c,e[i]=n,s=i}}}return t}return null}function _(e,t){var n=e.sortIndex-t.sortIndex;return 0!==n?n:e.id-t.id}var w=[],I=[],C=1,Q=null,R=3,D=!1,T=!1,O=!1;function P(e){for(var t=M(I);null!==t;){if(null===t.callback)k(I);else{if(!(t.startTime<=e))break;k(I),t.sortIndex=t.expirationTime,L(w,t)}t=M(I)}}function F(e){if(O=!1,P(e),!T)if(null!==M(w))T=!0,n(N);else{var t=M(I);null!==t&&s(F,t.startTime-e)}}function N(e,n){T=!1,O&&(O=!1,r()),D=!0;var a=R;try{for(P(n),Q=M(w);null!==Q&&(!(Q.expirationTime>n)||e&&!o());){var i=Q.callback;if(null!==i){Q.callback=null,R=Q.priorityLevel;var c=i(Q.expirationTime<=n);n=t.unstable_now(),"function"==typeof c?Q.callback=c:Q===M(w)&&k(w),P(n)}else k(w);Q=M(w)}if(null!==Q)var l=!0;else{var d=M(I);null!==d&&s(F,d.startTime-n),l=!1}return l}finally{Q=null,R=a,D=!1}}function B(e){switch(e){case 1:return-1;case 2:return 250;case 5:return 1073741823;case 4:return 1e4;default:return 5e3}}var V=a;t.unstable_IdlePriority=5,t.unstable_ImmediatePriority=1,t.unstable_LowPriority=4,t.unstable_NormalPriority=3,t.unstable_Profiling=null,t.unstable_UserBlockingPriority=2,t.unstable_cancelCallback=function(e){e.callback=null},t.unstable_continueExecution=function(){T||D||(T=!0,n(N))},t.unstable_getCurrentPriorityLevel=function(){return R},t.unstable_getFirstCallbackNode=function(){return M(w)},t.unstable_next=function(e){switch(R){case 1:case 2:case 3:var t=3;break;default:t=R}var n=R;R=t;try{return e()}finally{R=n}},t.unstable_pauseExecution=function(){},t.unstable_requestPaint=V,t.unstable_runWithPriority=function(e,t){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var n=R;R=e;try{return t()}finally{R=n}},t.unstable_scheduleCallback=function(e,o,a){var i=t.unstable_now();if("object"==typeof a&&null!==a){var c=a.delay;c="number"==typeof c&&0<c?i+c:i,a="number"==typeof a.timeout?a.timeout:B(e)}else a=B(e),c=i;return e={id:C++,callback:o,priorityLevel:e,startTime:c,expirationTime:a=c+a,sortIndex:-1},c>i?(e.sortIndex=c,L(I,e),null===M(w)&&e===M(I)&&(O?r():O=!0,s(F,c-i))):(e.sortIndex=a,L(w,e),T||D||(T=!0,n(N))),e},t.unstable_shouldYield=function(){var e=t.unstable_now();P(e);var n=M(w);return n!==Q&&null!==Q&&null!==n&&null!==n.callback&&n.startTime<=e&&n.expirationTime<Q.expirationTime||o()},t.unstable_wrapCallback=function(e){var t=R;return function(){var n=R;R=t;try{return e.apply(this,arguments)}finally{R=n}}}},63840:(e,t,n)=>{"use strict";e.exports=n(60053)},96774:e=>{e.exports=function(e,t,n,s){var r=n?n.call(s,e,t):void 0;if(void 0!==r)return!!r;if(e===t)return!0;if("object"!=typeof e||!e||"object"!=typeof t||!t)return!1;var o=Object.keys(e),a=Object.keys(t);if(o.length!==a.length)return!1;for(var i=Object.prototype.hasOwnProperty.bind(t),c=0;c<o.length;c++){var l=o[c];if(!i(l))return!1;var d=e[l],p=t[l];if(!1===(r=n?n.call(s,d,p,l):void 0)||void 0===r&&d!==p)return!1}return!0}},2177:(e,t,n)=>{"use strict";n.d(t,{Z:()=>r});var s="Invariant failed";function r(e,t){if(!e)throw new Error(s)}},53250:(e,t,n)=>{"use strict";var s=n(67294);var r="function"==typeof Object.is?Object.is:function(e,t){return e===t&&(0!==e||1/e==1/t)||e!=e&&t!=t},o=s.useState,a=s.useEffect,i=s.useLayoutEffect,c=s.useDebugValue;function l(e){var t=e.getSnapshot;e=e.value;try{var n=t();return!r(e,n)}catch(s){return!0}}var d="undefined"==typeof window||void 0===window.document||void 0===window.document.createElement?function(e,t){return t()}:function(e,t){var n=t(),s=o({inst:{value:n,getSnapshot:t}}),r=s[0].inst,d=s[1];return i((function(){r.value=n,r.getSnapshot=t,l(r)&&d({inst:r})}),[e,n,t]),a((function(){return l(r)&&d({inst:r}),e((function(){l(r)&&d({inst:r})}))}),[e]),c(n),n};t.useSyncExternalStore=void 0!==s.useSyncExternalStore?s.useSyncExternalStore:d},61688:(e,t,n)=>{"use strict";e.exports=n(53250)},36809:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>s});const s={title:"SynapseML",tagline:"Simple and Distributed Machine Learning",url:"https://microsoft.github.io",baseUrl:"/SynapseML/",favicon:"img/favicon.ico",organizationName:"microsoft",projectName:"SynapseML",trailingSlash:!0,customFields:{version:"1.0.4"},stylesheets:[{href:"https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css",integrity:"sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc",crossorigin:"anonymous"}],themeConfig:{prism:{theme:{plain:{color:"#393A34",backgroundColor:"#f6f8fa"},styles:[{types:["comment","prolog","doctype","cdata"],style:{color:"#999988",fontStyle:"italic"}},{types:["namespace"],style:{opacity:.7}},{types:["string","attr-value"],style:{color:"#e3116c"}},{types:["punctuation","operator"],style:{color:"#393A34"}},{types:["entity","url","symbol","number","boolean","variable","constant","property","regex","inserted"],style:{color:"#36acaa"}},{types:["atrule","keyword","attr-name","selector"],style:{color:"#00a4db"}},{types:["function","deleted","tag"],style:{color:"#d73a49"}},{types:["function-variable"],style:{color:"#6f42c1"}},{types:["tag","selector","keyword"],style:{color:"#00009f"}}]},darkTheme:{plain:{color:"#f8f8f2",backgroundColor:"#272822"},styles:[{types:["comment"],style:{color:"rgb(136, 132, 111)"}},{types:["string","changed"],style:{color:"rgb(230, 219, 116)"}},{types:["punctuation","tag","deleted"],style:{color:"rgb(249, 38, 114)"}},{types:["number","builtin"],style:{color:"rgb(174, 129, 255)"}},{types:["variable"],style:{color:"rgb(248, 248, 242)"}},{types:["function","attr-name","inserted"],style:{color:"rgb(166, 226, 46)"}}]},additionalLanguages:["csharp","powershell"],magicComments:[{className:"theme-code-block-highlighted-line",line:"highlight-next-line",block:{start:"highlight-start",end:"highlight-end"}}]},colorMode:{defaultMode:"dark",disableSwitch:!1,respectPrefersColorScheme:!1},image:"img/synapseml_og.jpg",navbar:{title:"SynapseML",logo:{alt:"SynapseML Logo",src:"img/logo.svg"},items:[{to:"docs/Overview",label:"Docs",position:"left"},{to:"blog",label:"Blog",position:"left"},{to:"videos",label:"Videos",position:"left"},{type:"docsVersionDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{type:"localeDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{label:"Developer Docs",position:"right",items:[{label:"Python",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html"},{label:"Scala",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html"}]},{href:"https://github.com/microsoft/SynapseML",position:"right",className:"header-github-link","aria-label":"GitHub repository"}],hideOnScroll:!1},footer:{style:"dark",links:[{title:"Docs",items:[{label:"Installation",to:"docs/Get%20Started/Install%20SynapseML"},{label:"Getting Started",to:"docs/Get%20Started/Quickstart%20-%20Your%20First%20Models"},{label:"Python API Reference",to:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html"},{label:"Scala API Reference",to:"https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html"}]},{title:"More",items:[{label:"Blog",to:"blog"},{label:"Videos",to:"videos"}]},{title:"Community",items:[{label:"GitHub",href:"https://github.com/microsoft/SynapseML"}]}],copyright:"Copyright \xa9 2024 Microsoft."},algolia:{appId:"GBW8AA15RD",apiKey:"70a6807005c645678741ab941bb89ed8",indexName:"synapseML",contextualSearch:!0,searchParameters:{},searchPagePath:"search"},announcementBar:{id:"announcementBar-1",content:'\u2b50\ufe0f If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> \u2b50',isCloseable:!0},docs:{versionPersistence:"localStorage",sidebar:{hideable:!1,autoCollapseCategories:!1}},metadata:[],tableOfContents:{minHeadingLevel:2,maxHeadingLevel:3}},presets:[["@docusaurus/preset-classic",{docs:{sidebarPath:"/home/vsts/work/1/s/website/sidebars.js",remarkPlugins:[null],rehypePlugins:[null]},theme:{customCss:"/home/vsts/work/1/s/website/src/css/custom.css"},gtag:{trackingID:"G-RWPE0183E8",anonymizeIP:!0},blog:{feedOptions:{type:"all"}}}]],plugins:[["@docusaurus/plugin-client-redirects",{redirects:[{to:"/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Create%20Audiobooks/"},{to:"/docs/Overview/",from:"/docs/about/"},{to:"/docs/Explore Algorithms/OpenAI/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20OpenAI/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/features/lightgbm/about/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/getting_started/installation/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",from:"/docs/features/isolation_forest/IsolationForest%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Quick Examples/transformers/transformers_cognitive/",from:"/docs/documentation/transformers/transformers_cognitive/"},{to:"/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",from:"/docs/features/onnx/ONNX%20-%20Inference%20on%20Spark/"},{to:"/docs/Explore Algorithms/AI Services/Geospatial Services/",from:"/docs/features/geospatial_services/GeospatialServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",from:"/docs/features/hyperparameter_tuning/HyperOpt-SynapseML/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Overview/",from:"/docs/features/vw/Vowpal%20Wabbit%20-%20Overview/"},{to:"/docs/Reference/Developer Setup/",from:"/docs/reference/developer-readme/"},{to:"/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/",from:"/docs/features/responsible_ai/Data%20Balance%20Analysis/"},{to:"/docs/Use with MLFlow/Overview/",from:"/docs/next/mlflow/examples/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/0.10.1/getting_started/installation/"},{to:"/docs/Explore Algorithms/Deep Learning/ONNX/",from:"/docs/next/features/onnx/about/"},{to:"/docs/Explore Algorithms/Causal Inference/Overview/",from:"/docs/features/causal_inference/about/"},{to:"/docs/Quick Examples/transformers/transformers_cognitive/",from:"/docs/next/documentation/transformers/transformers_cognitive/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",from:"/docs/next/features/isolation_forest/IsolationForest%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/next/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/getting_started/first_model/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",from:"/docs/next/features/onnx/ONNX%20-%20Inference%20on%20Spark/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/next/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",from:"/docs/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/"},{to:"/docs/Overview/",from:"/docs/next/about/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.11.0/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/getting_started/first_example/"},{to:"/docs/Use with MLFlow/Overview/",from:"/docs/mlflow/examples/"},{to:"/docs/Reference/Dotnet Setup/",from:"/docs/reference/dotnet-setup/"},{to:"/docs/Reference/Quickstart - LightGBM in Dotnet/",from:"/docs/0.10.0/getting_started/dotnet_example/"},{to:"/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",from:"/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/"},{to:"/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",from:"/docs/features/other/HyperParameterTuning%20-%20Fighting%20Breast%20Cancer/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/next/features/lightgbm/about/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/next/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Reference/Dotnet Setup/",from:"/docs/next/reference/dotnet-setup/"},{to:"/docs/Overview/",from:"/docs/0.10.0/about/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/0.10.0/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/OpenCV/Image Transformations/",from:"/docs/features/opencv/OpenCV%20-%20Pipeline%20Image%20Transformations/"},{to:"/docs/Overview/",from:"/docs/features/spark_serving/about/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.10.1/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.11.1/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.9.4/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/0.9.4/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",from:"/docs/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/"},{to:"/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/",from:"/docs/features/classification/Classification%20-%20Adult%20Census/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",from:"/docs/features/vw/Vowpal%20Wabbit%20-%20Contextual%20Bandits/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",from:"/docs/next/features/simple_deep_learning/DeepLearning%20-%20Deep%20Text%20Classification/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/next/getting_started/installation/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.10.1/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Use with MLFlow/Autologging/",from:"/docs/0.10.1/mlflow/autologging/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/0.9.4/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/0.9.5/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Quick Examples/estimators/estimators_cognitive/",from:"/docs/documentation/estimators/estimators_cognitive/"},{to:"/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Create%20a%20Multilingual%20Search%20Engine%20from%20Forms/"},{to:"/docs/Quick Examples/estimators/estimators_lightgbm/",from:"/docs/next/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/0.9.4/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/0.9.5/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Explore Algorithms/OpenAI/",from:"/docs/Explore%20Algorithms/OpenAI/"},{to:"/docs/Explore Algorithms/OpenCV/Image Transformations/",from:"/docs/documentation/transformers/transformers_opencv/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/0.10.0/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Use with MLFlow/Autologging/",from:"/docs/0.10.0/mlflow/autologging/"},{to:"/docs/Overview/",from:"/docs/0.11.0/about/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Overview/",from:"/docs/0.11.1/features/vw/Vowpal%20Wabbit%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",from:"/docs/0.9.4/features/responsible_ai/Data%20Balance%20Analysis/"},{to:"/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",from:"/docs/0.9.4/features/responsible_ai/DataBalanceAnalysis%20-%20Adult%20Census%20Income/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/Get%20Started/Install%20SynapseML/"},{to:"/docs/Quick Examples/transformers/transformers_core/",from:"/docs/next/documentation/transformers/transformers_core/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/Get%20Started/Quickstart%20-%20Your%20First%20Models/"},{to:"/docs/Explore Algorithms/Deep Learning/Getting Started/",from:"/docs/features/simple_deep_learning/about/"},{to:"/docs/Explore Algorithms/AI Services/Geospatial Services/",from:"/docs/next/features/geospatial_services/GeospatialServices%20-%20Overview/"}]}]],baseUrlIssueBanner:!0,i18n:{defaultLocale:"en",path:"i18n",locales:["en"],localeConfigs:{}},onBrokenLinks:"throw",onBrokenMarkdownLinks:"warn",onDuplicateRoutes:"warn",staticDirectories:["static"],themes:[],scripts:[],headTags:[],clientModules:[],titleDelimiter:"|",noIndex:!1,markdown:{mermaid:!1}}},83117:(e,t,n)=>{"use strict";function s(){return s=Object.assign?Object.assign.bind():function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},s.apply(this,arguments)}n.d(t,{Z:()=>s})},90144:(e,t,n)=>{"use strict";function s(e,t){return s=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},s(e,t)}function r(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,s(e,t)}n.d(t,{Z:()=>r})},80102:(e,t,n)=>{"use strict";function s(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||(r[n]=e[n]);return r}n.d(t,{Z:()=>s})},57529:e=>{"use strict";e.exports={}},16887:e=>{"use strict";e.exports=JSON.parse('{"/SynapseML/blog/-32c":{"__comp":"a6aa9e1f","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","items":[{"content":"c3c516ff"},{"content":"b72abe57"},{"content":"8cb11643"},{"content":"2b30bc14"},{"content":"ab9a8589"},{"content":"2d5b0e6a"}],"metadata":"6a95f87e"},"/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/-ecd":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"e1b2b5f3"},"/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/-870":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"e2ce57ed"},"/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/-83d":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"1abf1e18"},"/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/-347":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"b962e007"},"/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/-546":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"561bd03d"},"/SynapseML/blog/archive/-140":{"__comp":"9e4087bc","__context":{"plugin":"3fb29942"},"archive":"d175df5e"},"/SynapseML/blog/overview/-32f":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"cb5f1a06"},"/SynapseML/search/-a70":{"__comp":"1a4e3797","__context":{"plugin":"fc338b02"}},"/SynapseML/videos/-9fc":{"__comp":"34f00221","__context":{"plugin":"627aefca"},"config":"5e9f5e1a"},"/SynapseML/docs/0.11.3/-649":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"ffb7f88a"},"/SynapseML/docs/0.11.3/Deploy Models/Overview/-107":{"__comp":"17896441","content":"15a59c33"},"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/-7ca":{"__comp":"17896441","content":"37ab9beb"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-428":{"__comp":"17896441","content":"1e57c92a"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/-20d":{"__comp":"17896441","content":"6b9bdd6f"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-dd0":{"__comp":"17896441","content":"4bbbdfcf"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/-8b1":{"__comp":"17896441","content":"12d85ab5"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-7f9":{"__comp":"17896441","content":"47bbce76"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/-9df":{"__comp":"17896441","content":"cb7c2a83"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/-398":{"__comp":"17896441","content":"707d2a35"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-475":{"__comp":"17896441","content":"b80b493a"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-746":{"__comp":"17896441","content":"13f17286"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-00a":{"__comp":"17896441","content":"d25cde70"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-be4":{"__comp":"17896441","content":"900b935c"},"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-4ed":{"__comp":"17896441","content":"ae89cd3b"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/-f94":{"__comp":"17896441","content":"efbc69e1"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-af8":{"__comp":"17896441","content":"b8963dc0"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-c3d":{"__comp":"17896441","content":"93ba15e0"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-1f2":{"__comp":"17896441","content":"20641120"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/-d3c":{"__comp":"17896441","content":"fd4bd09d"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-b90":{"__comp":"17896441","content":"86ed1ff2"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-583":{"__comp":"17896441","content":"327be84b"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/-c90":{"__comp":"17896441","content":"04ee4ab4"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/-59d":{"__comp":"17896441","content":"3a4f9d93"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/-4d0":{"__comp":"17896441","content":"0f113696"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-b98":{"__comp":"17896441","content":"ba9c0924"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-52d":{"__comp":"17896441","content":"ee83122d"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-018":{"__comp":"17896441","content":"6ff7775f"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-086":{"__comp":"17896441","content":"deef0af8"},"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-2e6":{"__comp":"17896441","content":"ed316aed"},"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-cd5":{"__comp":"17896441","content":"6b40ec54"},"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/-7d2":{"__comp":"17896441","content":"391cb159"},"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-c77":{"__comp":"17896441","content":"e579bad6"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/-f47":{"__comp":"17896441","content":"e9b75d67"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/-ffd":{"__comp":"17896441","content":"e3089280"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-c37":{"__comp":"17896441","content":"5f17911b"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-3dc":{"__comp":"17896441","content":"6f8cd013"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/-5b7":{"__comp":"17896441","content":"da508cef"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/-e89":{"__comp":"17896441","content":"4e3910af"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-22c":{"__comp":"17896441","content":"b328a361"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-6da":{"__comp":"17896441","content":"4de42975"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-4f5":{"__comp":"17896441","content":"995576e9"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/-b53":{"__comp":"17896441","content":"1dc2f362"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/-e82":{"__comp":"17896441","content":"53a5cb1e"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-eb4":{"__comp":"17896441","content":"109f6864"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/-aa0":{"__comp":"17896441","content":"59ef8022"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/-547":{"__comp":"17896441","content":"052105dd"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/-ca9":{"__comp":"17896441","content":"6f479459"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-1c8":{"__comp":"17896441","content":"4997ef4f"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-850":{"__comp":"17896441","content":"5066efb2"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-103":{"__comp":"17896441","content":"c065fcb9"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-22a":{"__comp":"17896441","content":"255635fc"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/-4a3":{"__comp":"17896441","content":"79114dfe"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/-51c":{"__comp":"17896441","content":"50af03e4"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-79d":{"__comp":"17896441","content":"9b020c4a"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-dbd":{"__comp":"17896441","content":"fba72e87"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/-f6c":{"__comp":"17896441","content":"13246fd5"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-899":{"__comp":"17896441","content":"0fa36e32"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-de1":{"__comp":"17896441","content":"295a8e78"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-6e5":{"__comp":"17896441","content":"0606f1a5"},"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/-604":{"__comp":"17896441","content":"82878c84"},"/SynapseML/docs/0.11.3/Get Started/Install SynapseML/-53d":{"__comp":"17896441","content":"d3c9b0d7"},"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/-bcf":{"__comp":"17896441","content":"c4d09a44"},"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/-e8a":{"__comp":"17896441","content":"0a65b9de"},"/SynapseML/docs/0.11.3/Overview/-201":{"__comp":"17896441","content":"09bc1516"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/-f33":{"__comp":"17896441","content":"c40984d1"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/-d03":{"__comp":"17896441","content":"325e22bd"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/-0fc":{"__comp":"17896441","content":"5962ef6e"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/-113":{"__comp":"17896441","content":"4e044b4e"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/-bcf":{"__comp":"17896441","content":"a877f9e5"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/-3f4":{"__comp":"17896441","content":"0d6d64cd"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/-abc":{"__comp":"17896441","content":"9d095dba"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/-b7d":{"__comp":"17896441","content":"48329b94"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/-a66":{"__comp":"17896441","content":"ad10988d"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/-b96":{"__comp":"17896441","content":"548964ce"},"/SynapseML/docs/0.11.3/Reference/Contributor Guide/-06f":{"__comp":"17896441","content":"b41f295c"},"/SynapseML/docs/0.11.3/Reference/Developer Setup/-cfc":{"__comp":"17896441","content":"8e32e44d"},"/SynapseML/docs/0.11.3/Reference/Docker Setup/-419":{"__comp":"17896441","content":"57e687e8"},"/SynapseML/docs/0.11.3/Reference/Dotnet Setup/-46d":{"__comp":"17896441","content":"603706f6"},"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/-9b3":{"__comp":"17896441","content":"71042800"},"/SynapseML/docs/0.11.3/Reference/R Setup/-1a7":{"__comp":"17896441","content":"04fc65cc"},"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/-6a1":{"__comp":"17896441","content":"9c5088a2"},"/SynapseML/docs/0.11.3/Use with MLFlow/Install/-aa4":{"__comp":"17896441","content":"d39aa6d3"},"/SynapseML/docs/0.11.3/Use with MLFlow/Overview/-a43":{"__comp":"17896441","content":"1587ba1d"},"/SynapseML/docs/0.11.4/-72b":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"4e432063"},"/SynapseML/docs/0.11.4/Deploy Models/Overview/-1a4":{"__comp":"17896441","content":"a9330763"},"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/-ceb":{"__comp":"17896441","content":"ad139e3c"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-f98":{"__comp":"17896441","content":"cf57716c"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/-9e7":{"__comp":"17896441","content":"e21a3367"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-2c3":{"__comp":"17896441","content":"6e159789"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/-be8":{"__comp":"17896441","content":"77c259dd"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-1d4":{"__comp":"17896441","content":"fd3b6ea5"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/-31e":{"__comp":"17896441","content":"01a15f20"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/-8c4":{"__comp":"17896441","content":"74659d33"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-4f9":{"__comp":"17896441","content":"0637d1e7"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-d9d":{"__comp":"17896441","content":"f322d2da"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5a7":{"__comp":"17896441","content":"a406e231"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-e8f":{"__comp":"17896441","content":"e8b9c2c7"},"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-8e2":{"__comp":"17896441","content":"f8597cb0"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/-78b":{"__comp":"17896441","content":"8fd0c721"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-756":{"__comp":"17896441","content":"22f921cc"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-264":{"__comp":"17896441","content":"e4e95844"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-c8d":{"__comp":"17896441","content":"be95fd8f"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/-247":{"__comp":"17896441","content":"fe2c893b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-c00":{"__comp":"17896441","content":"b7802ae1"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-c42":{"__comp":"17896441","content":"aba932dd"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/-53f":{"__comp":"17896441","content":"ef6ec597"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/-2d6":{"__comp":"17896441","content":"8181e18a"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/-169":{"__comp":"17896441","content":"70b306ba"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-b68":{"__comp":"17896441","content":"67e06a8b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9c7":{"__comp":"17896441","content":"061b38c5"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-ff8":{"__comp":"17896441","content":"0daa5b3f"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-016":{"__comp":"17896441","content":"a34c49d0"},"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-6a6":{"__comp":"17896441","content":"e4ba4487"},"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-b11":{"__comp":"17896441","content":"4cee39d7"},"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/-e22":{"__comp":"17896441","content":"d9837698"},"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-934":{"__comp":"17896441","content":"a608660b"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/-fea":{"__comp":"17896441","content":"c5775233"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/-cfe":{"__comp":"17896441","content":"a0434473"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-78f":{"__comp":"17896441","content":"0453ae68"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-163":{"__comp":"17896441","content":"c413b43a"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/-29e":{"__comp":"17896441","content":"c73cfc52"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/-747":{"__comp":"17896441","content":"9fa7e8eb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-faf":{"__comp":"17896441","content":"bf002efe"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-566":{"__comp":"17896441","content":"0e16e9ea"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-491":{"__comp":"17896441","content":"d620a661"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/-7a4":{"__comp":"17896441","content":"4d31bfb3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/-75f":{"__comp":"17896441","content":"c991f47b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-2b0":{"__comp":"17896441","content":"15ea8379"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/-f95":{"__comp":"17896441","content":"60a2189a"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/-66b":{"__comp":"17896441","content":"7a1d9beb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/-e63":{"__comp":"17896441","content":"49e6864b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-783":{"__comp":"17896441","content":"31ee0fa3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-e01":{"__comp":"17896441","content":"db170a23"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-3b8":{"__comp":"17896441","content":"38d6824e"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-80b":{"__comp":"17896441","content":"069e3c4c"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/-18e":{"__comp":"17896441","content":"8328d740"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/-128":{"__comp":"17896441","content":"11c317cb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-02f":{"__comp":"17896441","content":"dea122e2"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-6d4":{"__comp":"17896441","content":"0e2bf69e"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/-b97":{"__comp":"17896441","content":"8af72580"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-6ca":{"__comp":"17896441","content":"ceeda143"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-c86":{"__comp":"17896441","content":"d41234a3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-964":{"__comp":"17896441","content":"b940e0b5"},"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/-cb4":{"__comp":"17896441","content":"c2492d56"},"/SynapseML/docs/0.11.4/Get Started/Install SynapseML/-34f":{"__comp":"17896441","content":"034e6169"},"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/-6fa":{"__comp":"17896441","content":"0e0ee9b2"},"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/-528":{"__comp":"17896441","content":"767a7177"},"/SynapseML/docs/0.11.4/Overview/-83a":{"__comp":"17896441","content":"b9f125cc"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/-32e":{"__comp":"17896441","content":"589adaf2"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/-269":{"__comp":"17896441","content":"965ed185"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/-11e":{"__comp":"17896441","content":"c90c942b"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/-c94":{"__comp":"17896441","content":"d1246f49"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/-bf0":{"__comp":"17896441","content":"1842239f"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/-728":{"__comp":"17896441","content":"dac82c00"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/-12e":{"__comp":"17896441","content":"e24ee294"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/-1cc":{"__comp":"17896441","content":"3ac37a06"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/-8fb":{"__comp":"17896441","content":"f7bfff73"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/-828":{"__comp":"17896441","content":"df01388f"},"/SynapseML/docs/0.11.4/Reference/Contributor Guide/-881":{"__comp":"17896441","content":"e9aab890"},"/SynapseML/docs/0.11.4/Reference/Developer Setup/-171":{"__comp":"17896441","content":"dd29718b"},"/SynapseML/docs/0.11.4/Reference/Docker Setup/-f85":{"__comp":"17896441","content":"e5c4ef01"},"/SynapseML/docs/0.11.4/Reference/Dotnet Setup/-f28":{"__comp":"17896441","content":"7aa7b8e0"},"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/-553":{"__comp":"17896441","content":"0b416bde"},"/SynapseML/docs/0.11.4/Reference/R Setup/-fd3":{"__comp":"17896441","content":"85aed2f9"},"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/-3cb":{"__comp":"17896441","content":"20e0fe38"},"/SynapseML/docs/0.11.4/Use with MLFlow/Install/-360":{"__comp":"17896441","content":"3c1f4383"},"/SynapseML/docs/0.11.4/Use with MLFlow/Overview/-4e1":{"__comp":"17896441","content":"d355047f"},"/SynapseML/docs/1.0.1/-c9e":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"cd8e97a5"},"/SynapseML/docs/1.0.1/Deploy Models/Overview/-7aa":{"__comp":"17896441","content":"aa5548ab"},"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/-c4a":{"__comp":"17896441","content":"179d57fa"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-d96":{"__comp":"17896441","content":"e1269457"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/-a96":{"__comp":"17896441","content":"ee6cea3b"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-4e5":{"__comp":"17896441","content":"4e105cba"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/-a84":{"__comp":"17896441","content":"70fe7dd7"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-e94":{"__comp":"17896441","content":"990f07de"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/-be5":{"__comp":"17896441","content":"7bf16cc6"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-76f":{"__comp":"17896441","content":"1155dc6f"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-5fa":{"__comp":"17896441","content":"4bea8531"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-417":{"__comp":"17896441","content":"6c7d288d"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-cb0":{"__comp":"17896441","content":"296054a7"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-831":{"__comp":"17896441","content":"ca0a1bad"},"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-ce6":{"__comp":"17896441","content":"166d99f5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/-a37":{"__comp":"17896441","content":"336404dc"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-0db":{"__comp":"17896441","content":"e887b273"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-d46":{"__comp":"17896441","content":"e7250a93"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-b74":{"__comp":"17896441","content":"00ab2cda"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/-109":{"__comp":"17896441","content":"2ece0fbe"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-de7":{"__comp":"17896441","content":"bece3771"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-250":{"__comp":"17896441","content":"ac581902"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/-68a":{"__comp":"17896441","content":"d252041b"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/-920":{"__comp":"17896441","content":"99e5ffa1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/-4a5":{"__comp":"17896441","content":"3b5de274"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-a53":{"__comp":"17896441","content":"e316de39"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9d3":{"__comp":"17896441","content":"39122aab"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-da8":{"__comp":"17896441","content":"5cfa133c"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-b25":{"__comp":"17896441","content":"a07d09c1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-18d":{"__comp":"17896441","content":"4db82253"},"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-f90":{"__comp":"17896441","content":"cff2e56e"},"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/-15c":{"__comp":"17896441","content":"993c5022"},"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-796":{"__comp":"17896441","content":"c2e40c34"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/-045":{"__comp":"17896441","content":"32fe2e34"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/-5e3":{"__comp":"17896441","content":"62983deb"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-6f5":{"__comp":"17896441","content":"6a26e359"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-457":{"__comp":"17896441","content":"7649ec47"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/-f73":{"__comp":"17896441","content":"e1ed1e14"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/-0f7":{"__comp":"17896441","content":"bef0d2d3"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-fa5":{"__comp":"17896441","content":"ed5c6c48"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-201":{"__comp":"17896441","content":"d3093636"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-bd7":{"__comp":"17896441","content":"3201258e"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/-979":{"__comp":"17896441","content":"99f8fee5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/-810":{"__comp":"17896441","content":"3cbfbf75"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-1e9":{"__comp":"17896441","content":"b0e28723"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/-6f0":{"__comp":"17896441","content":"90742975"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/-55c":{"__comp":"17896441","content":"00056db7"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/-e7f":{"__comp":"17896441","content":"b33598ac"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-df0":{"__comp":"17896441","content":"6eddee4c"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-94a":{"__comp":"17896441","content":"54287b47"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-41d":{"__comp":"17896441","content":"6c4c8509"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-d72":{"__comp":"17896441","content":"4130db01"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/-895":{"__comp":"17896441","content":"3a1147b5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/-b41":{"__comp":"17896441","content":"866b91a1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-c46":{"__comp":"17896441","content":"22444eb9"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-728":{"__comp":"17896441","content":"1a3608a9"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/-21f":{"__comp":"17896441","content":"12b89d4a"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-a43":{"__comp":"17896441","content":"f8aab3ca"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-7db":{"__comp":"17896441","content":"78aebd5d"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-9d2":{"__comp":"17896441","content":"c14fdd92"},"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/-697":{"__comp":"17896441","content":"423540f2"},"/SynapseML/docs/1.0.1/Get Started/Install SynapseML/-349":{"__comp":"17896441","content":"fc47fc53"},"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/-d39":{"__comp":"17896441","content":"90f00051"},"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/-2fa":{"__comp":"17896441","content":"0d432c84"},"/SynapseML/docs/1.0.1/Overview/-64e":{"__comp":"17896441","content":"1ad91055"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/-8d4":{"__comp":"17896441","content":"6a9ad78d"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/-d28":{"__comp":"17896441","content":"d9e5e528"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/-1b9":{"__comp":"17896441","content":"494501bd"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/-b23":{"__comp":"17896441","content":"445a1b28"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/-86c":{"__comp":"17896441","content":"e7bd83ca"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/-03e":{"__comp":"17896441","content":"001aea4c"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/-0db":{"__comp":"17896441","content":"fece3c24"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/-f4e":{"__comp":"17896441","content":"b38eec89"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/-945":{"__comp":"17896441","content":"a3fd5a6d"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/-369":{"__comp":"17896441","content":"ebf648dd"},"/SynapseML/docs/1.0.1/Reference/Contributor Guide/-5ab":{"__comp":"17896441","content":"76b137bf"},"/SynapseML/docs/1.0.1/Reference/Developer Setup/-971":{"__comp":"17896441","content":"b7c6295c"},"/SynapseML/docs/1.0.1/Reference/Docker Setup/-e6e":{"__comp":"17896441","content":"345903d4"},"/SynapseML/docs/1.0.1/Reference/Dotnet Setup/-179":{"__comp":"17896441","content":"5887dc62"},"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/-049":{"__comp":"17896441","content":"077ca2ec"},"/SynapseML/docs/1.0.1/Reference/R Setup/-392":{"__comp":"17896441","content":"b7787e72"},"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/-23e":{"__comp":"17896441","content":"f7580424"},"/SynapseML/docs/1.0.1/Use with MLFlow/Install/-0df":{"__comp":"17896441","content":"35826f14"},"/SynapseML/docs/1.0.1/Use with MLFlow/Overview/-d32":{"__comp":"17896441","content":"4f1ca6a9"},"/SynapseML/docs/1.0.2/-df4":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"dc1e40d7"},"/SynapseML/docs/1.0.2/Deploy Models/Overview/-a7c":{"__comp":"17896441","content":"e12b4691"},"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/-7f9":{"__comp":"17896441","content":"efc1978d"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-185":{"__comp":"17896441","content":"b1706689"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/-fa7":{"__comp":"17896441","content":"5c18deb5"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-1b3":{"__comp":"17896441","content":"5ef28062"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/-78e":{"__comp":"17896441","content":"d989c707"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-400":{"__comp":"17896441","content":"c23b5e26"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/-5da":{"__comp":"17896441","content":"fd748117"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-286":{"__comp":"17896441","content":"a34b09d4"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-c7f":{"__comp":"17896441","content":"514f485e"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-1d5":{"__comp":"17896441","content":"7c007ec6"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-d5d":{"__comp":"17896441","content":"b066233f"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-46e":{"__comp":"17896441","content":"f1b2dc7f"},"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-4ab":{"__comp":"17896441","content":"c34b1e36"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/-8e5":{"__comp":"17896441","content":"9cb6fa7c"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-405":{"__comp":"17896441","content":"093b81f4"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-2d7":{"__comp":"17896441","content":"00353f68"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-764":{"__comp":"17896441","content":"95d9e891"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/-f3b":{"__comp":"17896441","content":"f21c13c4"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-8c6":{"__comp":"17896441","content":"513002de"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-40b":{"__comp":"17896441","content":"e8d66fc1"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/-a43":{"__comp":"17896441","content":"b66e2100"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/-001":{"__comp":"17896441","content":"232d09b0"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/-a2e":{"__comp":"17896441","content":"6e71bda3"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-613":{"__comp":"17896441","content":"1e0d2d3b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-5c5":{"__comp":"17896441","content":"76d23901"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-893":{"__comp":"17896441","content":"bc4bf151"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-bb7":{"__comp":"17896441","content":"18c7d2a3"},"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-ccf":{"__comp":"17896441","content":"35bdc661"},"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-a9c":{"__comp":"17896441","content":"6c7ccb6d"},"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/-108":{"__comp":"17896441","content":"6bdbf6b8"},"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-a01":{"__comp":"17896441","content":"a8456d9f"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/-f2d":{"__comp":"17896441","content":"3e1e84da"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/-a4a":{"__comp":"17896441","content":"7c2e1100"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-a0a":{"__comp":"17896441","content":"16d05917"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-5bb":{"__comp":"17896441","content":"d0cb74df"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/-6ae":{"__comp":"17896441","content":"a2b5e5df"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/-4c7":{"__comp":"17896441","content":"1014102e"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-056":{"__comp":"17896441","content":"c38c658c"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-a63":{"__comp":"17896441","content":"c3b1d949"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-1e8":{"__comp":"17896441","content":"b75118f0"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/-6cd":{"__comp":"17896441","content":"1a215693"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/-318":{"__comp":"17896441","content":"af1e70b6"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-a37":{"__comp":"17896441","content":"75406112"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/-eed":{"__comp":"17896441","content":"80472ecc"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/-25a":{"__comp":"17896441","content":"18c9c2ba"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/-810":{"__comp":"17896441","content":"567cd2fd"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-5c9":{"__comp":"17896441","content":"e2689dfc"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-b99":{"__comp":"17896441","content":"3647ac9b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-3dd":{"__comp":"17896441","content":"4ef2f7cf"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-1bd":{"__comp":"17896441","content":"88b63415"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/-0b3":{"__comp":"17896441","content":"b576c645"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/-250":{"__comp":"17896441","content":"c2e57a1b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-bd7":{"__comp":"17896441","content":"d535b358"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-b2f":{"__comp":"17896441","content":"403e642d"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/-5a7":{"__comp":"17896441","content":"445012ce"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-4ab":{"__comp":"17896441","content":"ff428354"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-d16":{"__comp":"17896441","content":"03d0d544"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-157":{"__comp":"17896441","content":"73cf52e8"},"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/-3f8":{"__comp":"17896441","content":"a0d8a1a2"},"/SynapseML/docs/1.0.2/Get Started/Install SynapseML/-01e":{"__comp":"17896441","content":"98258c5e"},"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/-0aa":{"__comp":"17896441","content":"268b7b40"},"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/-095":{"__comp":"17896441","content":"92ce4fd5"},"/SynapseML/docs/1.0.2/Overview/-c78":{"__comp":"17896441","content":"b4898d44"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/-348":{"__comp":"17896441","content":"6c61f93b"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/-386":{"__comp":"17896441","content":"56d039a1"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/-585":{"__comp":"17896441","content":"8462374b"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/-fa7":{"__comp":"17896441","content":"f14633e7"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/-b73":{"__comp":"17896441","content":"3e9a7422"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/-762":{"__comp":"17896441","content":"10e13725"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/-7ec":{"__comp":"17896441","content":"f38a0cbd"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/-bad":{"__comp":"17896441","content":"d0c5e4fa"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/-183":{"__comp":"17896441","content":"39582d99"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/-029":{"__comp":"17896441","content":"07f6b90a"},"/SynapseML/docs/1.0.2/Reference/Contributor Guide/-8f1":{"__comp":"17896441","content":"7b60d8e3"},"/SynapseML/docs/1.0.2/Reference/Developer Setup/-5bf":{"__comp":"17896441","content":"9e5b788f"},"/SynapseML/docs/1.0.2/Reference/Docker Setup/-294":{"__comp":"17896441","content":"1789daeb"},"/SynapseML/docs/1.0.2/Reference/Dotnet Setup/-b72":{"__comp":"17896441","content":"012c1c0f"},"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/-595":{"__comp":"17896441","content":"ef77ef39"},"/SynapseML/docs/1.0.2/Reference/R Setup/-a22":{"__comp":"17896441","content":"74091a62"},"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/-e9d":{"__comp":"17896441","content":"25fb872c"},"/SynapseML/docs/1.0.2/Use with MLFlow/Install/-7d6":{"__comp":"17896441","content":"1f31a275"},"/SynapseML/docs/1.0.2/Use with MLFlow/Overview/-e3c":{"__comp":"17896441","content":"55203621"},"/SynapseML/docs/1.0.3/-60e":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"66d63bfc"},"/SynapseML/docs/1.0.3/Deploy Models/Overview/-1cf":{"__comp":"17896441","content":"ddacd194"},"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/-695":{"__comp":"17896441","content":"6e2678b7"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-0c3":{"__comp":"17896441","content":"fe5d19cb"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/-e88":{"__comp":"17896441","content":"321ba9c5"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-6d7":{"__comp":"17896441","content":"4b18e5ae"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/-0d9":{"__comp":"17896441","content":"d36f3429"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-5bc":{"__comp":"17896441","content":"2c5f0abd"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/-3ee":{"__comp":"17896441","content":"6e4e5cd9"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-7b8":{"__comp":"17896441","content":"51878b77"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-5f9":{"__comp":"17896441","content":"b35d004b"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-85d":{"__comp":"17896441","content":"d81d1137"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-6e8":{"__comp":"17896441","content":"2e93ae0f"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-e1a":{"__comp":"17896441","content":"b739c60a"},"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-54d":{"__comp":"17896441","content":"556ad472"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/-562":{"__comp":"17896441","content":"59d9fecc"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-734":{"__comp":"17896441","content":"5407376f"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-410":{"__comp":"17896441","content":"3c22aa66"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-823":{"__comp":"17896441","content":"0a9c6347"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/-1ea":{"__comp":"17896441","content":"1293b2b0"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-96b":{"__comp":"17896441","content":"943afd32"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-432":{"__comp":"17896441","content":"08019edc"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/-f57":{"__comp":"17896441","content":"01587ade"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/-141":{"__comp":"17896441","content":"f3e986ee"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/-005":{"__comp":"17896441","content":"328f3af4"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-2d6":{"__comp":"17896441","content":"888e2229"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-abe":{"__comp":"17896441","content":"3e7c5569"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-fb6":{"__comp":"17896441","content":"1a62d1af"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-1fe":{"__comp":"17896441","content":"c4d68582"},"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-561":{"__comp":"17896441","content":"a563fa93"},"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-158":{"__comp":"17896441","content":"cee4c3b9"},"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/-e1b":{"__comp":"17896441","content":"7ad384dc"},"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-99d":{"__comp":"17896441","content":"5a3e1bca"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/-1f6":{"__comp":"17896441","content":"1518f45a"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/-b8d":{"__comp":"17896441","content":"c764f2c0"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-3b4":{"__comp":"17896441","content":"d7f58694"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-146":{"__comp":"17896441","content":"63cd8b22"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/-57d":{"__comp":"17896441","content":"88f8e3e1"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/-e16":{"__comp":"17896441","content":"e03f5a54"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-3d8":{"__comp":"17896441","content":"c3dac62d"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-30f":{"__comp":"17896441","content":"3a7a88af"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-e7e":{"__comp":"17896441","content":"fe6131e8"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/-8c5":{"__comp":"17896441","content":"b3272230"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/-1f6":{"__comp":"17896441","content":"75514f6a"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-ea9":{"__comp":"17896441","content":"b66ab6b5"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/-211":{"__comp":"17896441","content":"5dbf4619"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/-18d":{"__comp":"17896441","content":"980b00ea"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/-d3d":{"__comp":"17896441","content":"f027bb15"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-025":{"__comp":"17896441","content":"de3bb7c3"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-878":{"__comp":"17896441","content":"16d8ca91"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-326":{"__comp":"17896441","content":"f5b4e517"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-a63":{"__comp":"17896441","content":"e5fe80e3"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/-8ed":{"__comp":"17896441","content":"d84904f9"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/-893":{"__comp":"17896441","content":"455aa1bd"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-747":{"__comp":"17896441","content":"82d7cc79"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-484":{"__comp":"17896441","content":"3ea19ec0"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/-60f":{"__comp":"17896441","content":"7aa49360"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-c16":{"__comp":"17896441","content":"f537d1e9"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-35d":{"__comp":"17896441","content":"0713a501"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-f74":{"__comp":"17896441","content":"4801d370"},"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/-ed3":{"__comp":"17896441","content":"f228fdc5"},"/SynapseML/docs/1.0.3/Get Started/Install SynapseML/-df3":{"__comp":"17896441","content":"2d8a44d7"},"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/-450":{"__comp":"17896441","content":"ac39f7dc"},"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/-dff":{"__comp":"17896441","content":"83203dd1"},"/SynapseML/docs/1.0.3/Overview/-094":{"__comp":"17896441","content":"39a50b22"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/-505":{"__comp":"17896441","content":"cf666d49"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/-6ff":{"__comp":"17896441","content":"9688a5be"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/-104":{"__comp":"17896441","content":"faca745a"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/-08b":{"__comp":"17896441","content":"bc85b804"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/-74d":{"__comp":"17896441","content":"10b7acf6"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/-baf":{"__comp":"17896441","content":"e402e26d"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/-dcc":{"__comp":"17896441","content":"75801678"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/-80b":{"__comp":"17896441","content":"4585b70e"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/-4af":{"__comp":"17896441","content":"c39e3602"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/-9cd":{"__comp":"17896441","content":"604b2d50"},"/SynapseML/docs/1.0.3/Reference/Contributor Guide/-676":{"__comp":"17896441","content":"275c0efd"},"/SynapseML/docs/1.0.3/Reference/Developer Setup/-904":{"__comp":"17896441","content":"ed40d5f1"},"/SynapseML/docs/1.0.3/Reference/Docker Setup/-963":{"__comp":"17896441","content":"3ee83f58"},"/SynapseML/docs/1.0.3/Reference/Dotnet Setup/-c7f":{"__comp":"17896441","content":"09990b79"},"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/-9f9":{"__comp":"17896441","content":"f0649d54"},"/SynapseML/docs/1.0.3/Reference/R Setup/-3a0":{"__comp":"17896441","content":"d57515f7"},"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/-00b":{"__comp":"17896441","content":"b35e9998"},"/SynapseML/docs/1.0.3/Use with MLFlow/Install/-5a1":{"__comp":"17896441","content":"17f418b7"},"/SynapseML/docs/1.0.3/Use with MLFlow/Overview/-234":{"__comp":"17896441","content":"8a1bca9d"},"/SynapseML/docs/next/-b5f":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"935f2afb"},"/SynapseML/docs/next/Deploy Models/Overview/-29d":{"__comp":"17896441","content":"94d743d6"},"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/-eeb":{"__comp":"17896441","content":"2c98ca3e"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-12a":{"__comp":"17896441","content":"0349e4f0"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/-00a":{"__comp":"17896441","content":"318ccb43"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-e9b":{"__comp":"17896441","content":"e464d112"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/-0d4":{"__comp":"17896441","content":"fa2e2688"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-6d0":{"__comp":"17896441","content":"0e3f998d"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/-bef":{"__comp":"17896441","content":"57ef17c5"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-4f8":{"__comp":"17896441","content":"9530a2cf"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-3a8":{"__comp":"17896441","content":"1b7af47d"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-385":{"__comp":"17896441","content":"c0391845"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5c7":{"__comp":"17896441","content":"97592aac"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-f4f":{"__comp":"17896441","content":"0a954229"},"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-bf9":{"__comp":"17896441","content":"b4a95996"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/-89c":{"__comp":"17896441","content":"b83021be"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-fc9":{"__comp":"17896441","content":"d707d2ed"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-80d":{"__comp":"17896441","content":"0fb98d33"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/-b09":{"__comp":"17896441","content":"58b8f176"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-12f":{"__comp":"17896441","content":"3bed3e2f"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/-86b":{"__comp":"17896441","content":"41f327fc"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-25e":{"__comp":"17896441","content":"2bcda99e"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-c31":{"__comp":"17896441","content":"1e14124f"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/-1a6":{"__comp":"17896441","content":"5e7c3303"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/-270":{"__comp":"17896441","content":"42736d5f"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/-c9a":{"__comp":"17896441","content":"ea2d2bcc"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-fdc":{"__comp":"17896441","content":"6a07b55e"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9cf":{"__comp":"17896441","content":"90e4432b"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-a7b":{"__comp":"17896441","content":"5eddefdd"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-641":{"__comp":"17896441","content":"bd1bcb2c"},"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-6e0":{"__comp":"17896441","content":"40b49758"},"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-8d1":{"__comp":"17896441","content":"905ce8b5"},"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/-50f":{"__comp":"17896441","content":"1d820955"},"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-d9a":{"__comp":"17896441","content":"5225b7e0"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/-ef5":{"__comp":"17896441","content":"96cf5ff0"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/-43a":{"__comp":"17896441","content":"ed6d544d"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/-ed1":{"__comp":"17896441","content":"00e0a48e"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/-a4b":{"__comp":"17896441","content":"dd3f9ada"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-a86":{"__comp":"17896441","content":"52880d18"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-101":{"__comp":"17896441","content":"120352d6"},"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/-5d7":{"__comp":"17896441","content":"83586bff"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/-1e7":{"__comp":"17896441","content":"d8b3ac6a"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-88e":{"__comp":"17896441","content":"9c279ae7"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-d5d":{"__comp":"17896441","content":"9da9112f"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-41e":{"__comp":"17896441","content":"91e9cb67"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/-d73":{"__comp":"17896441","content":"2e5f4263"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/-267":{"__comp":"17896441","content":"31bb7274"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-95b":{"__comp":"17896441","content":"66fcb786"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/-798":{"__comp":"17896441","content":"69dc6d0a"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/-0ec":{"__comp":"17896441","content":"51dbb02f"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/-039":{"__comp":"17896441","content":"c5c95e45"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-2fd":{"__comp":"17896441","content":"2137a7cd"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-8f3":{"__comp":"17896441","content":"b5375b6f"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-484":{"__comp":"17896441","content":"2406587d"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-3d2":{"__comp":"17896441","content":"bc6a9944"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/-caf":{"__comp":"17896441","content":"48cf7354"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/-219":{"__comp":"17896441","content":"d13e532e"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-b35":{"__comp":"17896441","content":"a81cf8b0"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-c0f":{"__comp":"17896441","content":"2d527871"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/-fae":{"__comp":"17896441","content":"610d430a"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-3c9":{"__comp":"17896441","content":"33b3776b"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-71e":{"__comp":"17896441","content":"41ee0cff"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-e9d":{"__comp":"17896441","content":"8eb438b4"},"/SynapseML/docs/next/Get Started/Create a Spark Cluster/-37f":{"__comp":"17896441","content":"a19ce767"},"/SynapseML/docs/next/Get Started/Install SynapseML/-975":{"__comp":"17896441","content":"73c2022c"},"/SynapseML/docs/next/Get Started/Quickstart - Your First Models/-8a6":{"__comp":"17896441","content":"283b8f99"},"/SynapseML/docs/next/Get Started/Set up Cognitive Services/-47d":{"__comp":"17896441","content":"b2be3901"},"/SynapseML/docs/next/Overview/-9a5":{"__comp":"17896441","content":"cbc3190d"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/-c47":{"__comp":"17896441","content":"521f9727"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/-6e1":{"__comp":"17896441","content":"34ff21b6"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_core/-7e6":{"__comp":"17896441","content":"9783ff24"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/-5eb":{"__comp":"17896441","content":"792eae7e"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/-49f":{"__comp":"17896441","content":"88e44e99"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/-443":{"__comp":"17896441","content":"1b3ab3b8"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_core/-0a9":{"__comp":"17896441","content":"bb7a48da"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/-137":{"__comp":"17896441","content":"f7a32432"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/-3f4":{"__comp":"17896441","content":"69bb4b11"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/-d93":{"__comp":"17896441","content":"8590feaa"},"/SynapseML/docs/next/Reference/Contributor Guide/-dec":{"__comp":"17896441","content":"104bf0a6"},"/SynapseML/docs/next/Reference/Developer Setup/-1c5":{"__comp":"17896441","content":"1db1f785"},"/SynapseML/docs/next/Reference/Docker Setup/-598":{"__comp":"17896441","content":"645abc7d"},"/SynapseML/docs/next/Reference/Dotnet Setup/-ded":{"__comp":"17896441","content":"f85db098"},"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/-8a8":{"__comp":"17896441","content":"5b779334"},"/SynapseML/docs/next/Reference/R Setup/-d8b":{"__comp":"17896441","content":"44ff0d07"},"/SynapseML/docs/next/Use with MLFlow/Autologging/-0ca":{"__comp":"17896441","content":"c95882d3"},"/SynapseML/docs/next/Use with MLFlow/Install/-9b7":{"__comp":"17896441","content":"fbb011bb"},"/SynapseML/docs/next/Use with MLFlow/Overview/-b10":{"__comp":"17896441","content":"a6191053"},"/SynapseML/docs/-352":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"76bf568a"},"/SynapseML/docs/Deploy Models/Overview/-438":{"__comp":"17896441","content":"3b75a1c6"},"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/-e9f":{"__comp":"17896441","content":"3d963cd5"},"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-3f1":{"__comp":"17896441","content":"4f81391a"},"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/-c3c":{"__comp":"17896441","content":"d05981ec"},"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-de7":{"__comp":"17896441","content":"56543a15"},"/SynapseML/docs/Explore Algorithms/AI Services/Overview/-52c":{"__comp":"17896441","content":"8bff2444"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-dc7":{"__comp":"17896441","content":"50f80512"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/-2c5":{"__comp":"17896441","content":"a1824f78"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-f90":{"__comp":"17896441","content":"3dafa002"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-425":{"__comp":"17896441","content":"0eaf8468"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-d37":{"__comp":"17896441","content":"dc7f7ac1"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5e1":{"__comp":"17896441","content":"1766722a"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-eb1":{"__comp":"17896441","content":"8a46fce0"},"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-7e3":{"__comp":"17896441","content":"efbb59fb"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/-75a":{"__comp":"17896441","content":"23af8e3e"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-b1b":{"__comp":"17896441","content":"e863531f"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-cf3":{"__comp":"17896441","content":"8d990105"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/-bcf":{"__comp":"17896441","content":"7aeb6f16"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-3e2":{"__comp":"17896441","content":"7296bed0"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/-618":{"__comp":"17896441","content":"368de161"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-7c2":{"__comp":"17896441","content":"27de0303"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-a7a":{"__comp":"17896441","content":"3ef20ed6"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/-1ba":{"__comp":"17896441","content":"e0077969"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/-628":{"__comp":"17896441","content":"1f6f5ac9"},"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/-90b":{"__comp":"17896441","content":"59e4d88e"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-57b":{"__comp":"17896441","content":"8799ef55"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-b77":{"__comp":"17896441","content":"6094981e"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-a81":{"__comp":"17896441","content":"38b375cc"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-93c":{"__comp":"17896441","content":"78cb5959"},"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-87a":{"__comp":"17896441","content":"a5ffe972"},"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-699":{"__comp":"17896441","content":"b4ae214b"},"/SynapseML/docs/Explore Algorithms/LightGBM/Overview/-300":{"__comp":"17896441","content":"141dc4a8"},"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-5c1":{"__comp":"17896441","content":"2fb6c1b0"},"/SynapseML/docs/Explore Algorithms/OpenAI/-06d":{"__comp":"17896441","content":"d1428557"},"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/-3b0":{"__comp":"17896441","content":"2461af4d"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/-b8b":{"__comp":"17896441","content":"c76e8cac"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-d6d":{"__comp":"17896441","content":"c82134d8"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-baa":{"__comp":"17896441","content":"7275f11a"},"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/-179":{"__comp":"17896441","content":"d95a70ea"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/-4b2":{"__comp":"17896441","content":"2cec6bcf"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-54a":{"__comp":"17896441","content":"f264933a"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-572":{"__comp":"17896441","content":"eed630e0"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-743":{"__comp":"17896441","content":"6c82e982"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/-82e":{"__comp":"17896441","content":"bf04857e"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/-dc2":{"__comp":"17896441","content":"affaa541"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-72f":{"__comp":"17896441","content":"eb0ef952"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/-e8d":{"__comp":"17896441","content":"014ce9a3"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/-e6c":{"__comp":"17896441","content":"ce7af6ae"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/-f44":{"__comp":"17896441","content":"3bbe0f34"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-9d9":{"__comp":"17896441","content":"26631026"},"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-082":{"__comp":"17896441","content":"5e008bd1"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-f62":{"__comp":"17896441","content":"b92ffdc6"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-804":{"__comp":"17896441","content":"5efd5006"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/-60b":{"__comp":"17896441","content":"8ff6772d"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/-3c5":{"__comp":"17896441","content":"644ff257"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-32b":{"__comp":"17896441","content":"dfb6dfe9"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-046":{"__comp":"17896441","content":"b7226801"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/-5b3":{"__comp":"17896441","content":"fc598b5d"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-490":{"__comp":"17896441","content":"56087ddb"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-5dd":{"__comp":"17896441","content":"01eb996b"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-804":{"__comp":"17896441","content":"fa889bee"},"/SynapseML/docs/Get Started/Create a Spark Cluster/-37d":{"__comp":"17896441","content":"cc52e3cd"},"/SynapseML/docs/Get Started/Install SynapseML/-16c":{"__comp":"17896441","content":"62ccfbde"},"/SynapseML/docs/Get Started/Quickstart - Your First Models/-e0c":{"__comp":"17896441","content":"e84755e1"},"/SynapseML/docs/Get Started/Set up Cognitive Services/-ea8":{"__comp":"17896441","content":"ce40236b"},"/SynapseML/docs/Overview/-58e":{"__comp":"17896441","content":"bf8a5c8a"},"/SynapseML/docs/Quick Examples/estimators/estimators_causal/-e9d":{"__comp":"17896441","content":"104b9b56"},"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/-419":{"__comp":"17896441","content":"79d79762"},"/SynapseML/docs/Quick Examples/estimators/estimators_core/-63c":{"__comp":"17896441","content":"421d486c"},"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/-d43":{"__comp":"17896441","content":"b28fe732"},"/SynapseML/docs/Quick Examples/estimators/estimators_vw/-bd9":{"__comp":"17896441","content":"1ef65056"},"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/-bbd":{"__comp":"17896441","content":"408524d4"},"/SynapseML/docs/Quick Examples/transformers/transformers_core/-1a1":{"__comp":"17896441","content":"5bf420e9"},"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/-483":{"__comp":"17896441","content":"5c0b440a"},"/SynapseML/docs/Quick Examples/transformers/transformers_opencv/-8a0":{"__comp":"17896441","content":"513214d5"},"/SynapseML/docs/Quick Examples/transformers/transformers_vw/-fd7":{"__comp":"17896441","content":"21f47424"},"/SynapseML/docs/Reference/Contributor Guide/-6e9":{"__comp":"17896441","content":"294c060e"},"/SynapseML/docs/Reference/Developer Setup/-367":{"__comp":"17896441","content":"b88f1f9b"},"/SynapseML/docs/Reference/Docker Setup/-a2a":{"__comp":"17896441","content":"405fabd0"},"/SynapseML/docs/Reference/Dotnet Setup/-995":{"__comp":"17896441","content":"58486a98"},"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/-e58":{"__comp":"17896441","content":"4631a032"},"/SynapseML/docs/Reference/R Setup/-d4d":{"__comp":"17896441","content":"04804d02"},"/SynapseML/docs/Use with MLFlow/Autologging/-1cd":{"__comp":"17896441","content":"2a023663"},"/SynapseML/docs/Use with MLFlow/Install/-a7e":{"__comp":"17896441","content":"70e8008e"},"/SynapseML/docs/Use with MLFlow/Overview/-424":{"__comp":"17896441","content":"15516451"},"/SynapseML/-6e1":{"__comp":"c4f5d8e4","__context":{"plugin":"627aefca"},"config":"5e9f5e1a"}}')}},e=>{e.O(0,[40532],(()=>{return t=49383,e(e.s=t);var t}));e.O()}]);
\ No newline at end of file
diff --git a/assets/js/main.f50c121d.js.LICENSE.txt b/assets/js/main.acd258f6.js.LICENSE.txt
similarity index 100%
rename from assets/js/main.f50c121d.js.LICENSE.txt
rename to assets/js/main.acd258f6.js.LICENSE.txt
diff --git a/assets/js/main.f50c121d.js b/assets/js/main.f50c121d.js
deleted file mode 100644
index aeb370bfb9..0000000000
--- a/assets/js/main.f50c121d.js
+++ /dev/null
@@ -1,2 +0,0 @@
-/*! For license information please see main.f50c121d.js.LICENSE.txt */
-(self.webpackChunksynapseml=self.webpackChunksynapseml||[]).push([[40179],{20830:(e,t,n)=>{"use strict";n.d(t,{W:()=>r});var s=n(67294);function r(){return s.createElement("svg",{width:"20",height:"20",className:"DocSearch-Search-Icon",viewBox:"0 0 20 20"},s.createElement("path",{d:"M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z",stroke:"currentColor",fill:"none",fillRule:"evenodd",strokeLinecap:"round",strokeLinejoin:"round"}))}},723:(e,t,n)=>{"use strict";n.d(t,{Z:()=>m});var s=n(67294),r=n(83117),o=n(68356),a=n.n(o),i=n(16887);const c={"00056db7":[()=>n.e(81663).then(n.bind(n,83477)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard.md",83477],"001aea4c":[()=>Promise.all([n.e(40532),n.e(390)]).then(n.bind(n,5642)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_cognitive.md",5642],"00353f68":[()=>n.e(79043).then(n.bind(n,82885)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",82885],"00ab2cda":[()=>n.e(44334).then(n.bind(n,19118)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",19118],"012c1c0f":[()=>Promise.all([n.e(40532),n.e(38261)]).then(n.bind(n,45919)),"@site/versioned_docs/version-1.0.2/Reference/Dotnet Setup.md",45919],"014ce9a3":[()=>n.e(10643).then(n.bind(n,63969)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Data Balance Analysis.md",63969],"01587ade":[()=>n.e(33988).then(n.bind(n,1059)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Distributed Training.md",1059],"01a15f20":[()=>n.e(81970).then(n.bind(n,53471)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",53471],"01eb996b":[()=>n.e(47552).then(n.bind(n,12231)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",12231],"0349e4f0":[()=>n.e(58524).then(n.bind(n,14930)),"@site/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",14930],"034e6169":[()=>n.e(43730).then(n.bind(n,28641)),"@site/versioned_docs/version-0.11.4/Get Started/Install SynapseML.md",28641],"03d0d544":[()=>n.e(26676).then(n.bind(n,46814)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",46814],"0453ae68":[()=>n.e(35939).then(n.bind(n,18496)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",18496],"04804d02":[()=>n.e(41607).then(n.bind(n,17044)),"@site/versioned_docs/version-1.0.4/Reference/R Setup.md",17044],"04ee4ab4":[()=>n.e(62217).then(n.bind(n,41680)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Distributed Training.md",41680],"04fc65cc":[()=>n.e(61755).then(n.bind(n,92078)),"@site/versioned_docs/version-0.11.3/Reference/R Setup.md",92078],"052105dd":[()=>n.e(23036).then(n.bind(n,47333)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard.md",47333],"0606f1a5":[()=>n.e(32789).then(n.bind(n,80)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",80],"061b38c5":[()=>n.e(35124).then(n.bind(n,60062)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",60062],"0637d1e7":[()=>n.e(35750).then(n.bind(n,95457)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",95457],"069e3c4c":[()=>n.e(50745).then(n.bind(n,64206)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",64206],"0713a501":[()=>n.e(37592).then(n.bind(n,9134)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",9134],"077ca2ec":[()=>n.e(37610).then(n.bind(n,9661)),"@site/versioned_docs/version-1.0.1/Reference/Quickstart - LightGBM in Dotnet.md",9661],"07f6b90a":[()=>Promise.all([n.e(40532),n.e(15839)]).then(n.bind(n,55678)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_vw.md",55678],"08019edc":[()=>n.e(62773).then(n.bind(n,55377)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",55377],"093b81f4":[()=>n.e(99970).then(n.bind(n,86786)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",86786],"09990b79":[()=>Promise.all([n.e(40532),n.e(97800)]).then(n.bind(n,88602)),"@site/versioned_docs/version-1.0.3/Reference/Dotnet Setup.md",88602],"09bc1516":[()=>n.e(2408).then(n.bind(n,22601)),"@site/versioned_docs/version-0.11.3/Overview.md",22601],"0a65b9de":[()=>n.e(939).then(n.bind(n,76455)),"@site/versioned_docs/version-0.11.3/Get Started/Set up Cognitive Services.md",76455],"0a954229":[()=>n.e(79088).then(n.bind(n,4489)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",4489],"0a9c6347":[()=>n.e(30460).then(n.bind(n,50140)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",50140],"0b416bde":[()=>n.e(78579).then(n.bind(n,76262)),"@site/versioned_docs/version-0.11.4/Reference/Quickstart - LightGBM in Dotnet.md",76262],"0d432c84":[()=>n.e(34797).then(n.bind(n,72089)),"@site/versioned_docs/version-1.0.1/Get Started/Set up Cognitive Services.md",72089],"0d6d64cd":[()=>Promise.all([n.e(40532),n.e(39307)]).then(n.bind(n,87804)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_cognitive.md",87804],"0daa5b3f":[()=>n.e(35024).then(n.bind(n,73426)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",73426],"0e0ee9b2":[()=>n.e(92045).then(n.bind(n,29119)),"@site/versioned_docs/version-0.11.4/Get Started/Quickstart - Your First Models.md",29119],"0e16e9ea":[()=>n.e(97721).then(n.bind(n,37562)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",37562],"0e2bf69e":[()=>n.e(94395).then(n.bind(n,87433)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",87433],"0e3f998d":[()=>n.e(64482).then(n.bind(n,67479)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",67479],"0eaf8468":[()=>n.e(26184).then(n.bind(n,74137)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",74137],"0f113696":[()=>n.e(16030).then(n.bind(n,78281)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/ONNX.md",78281],"0fa36e32":[()=>n.e(66555).then(n.bind(n,33171)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",33171],"0fb98d33":[()=>n.e(46338).then(n.bind(n,63159)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",63159],"1014102e":[()=>n.e(94744).then(n.bind(n,1110)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Cyber ML.md",1110],"104b9b56":[()=>Promise.all([n.e(40532),n.e(45282)]).then(n.bind(n,90171)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_causal.md",90171],"104bf0a6":[()=>n.e(68314).then(n.bind(n,15734)),"@site/docs/Reference/Contributor Guide.md",15734],"109f6864":[()=>n.e(94994).then(n.bind(n,95820)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",95820],"10b7acf6":[()=>Promise.all([n.e(40532),n.e(6004)]).then(n.bind(n,40044)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_vw.md",40044],"10e13725":[()=>Promise.all([n.e(40532),n.e(25116)]).then(n.bind(n,34855)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_cognitive.md",34855],"1155dc6f":[()=>n.e(5789).then(n.bind(n,74766)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",74766],"11c317cb":[()=>n.e(18933).then(n.bind(n,24433)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Text Explainers.md",24433],"120352d6":[()=>n.e(89262).then(n.bind(n,17608)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",17608],"1293b2b0":[()=>n.e(57074).then(n.bind(n,73303)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier.md",73303],"12b89d4a":[()=>n.e(52804).then(n.bind(n,44536)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Overview.md",44536],"12d85ab5":[()=>n.e(82412).then(n.bind(n,11105)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Overview.md",11105],"13246fd5":[()=>n.e(48807).then(n.bind(n,13869)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Overview.md",13869],"13f17286":[()=>n.e(72912).then(n.bind(n,38060)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",38060],"141dc4a8":[()=>n.e(34334).then(n.bind(n,72356)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/LightGBM/Overview.md",72356],"1518f45a":[()=>n.e(9296).then(n.bind(n,54429)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/OpenAI.md",54429],15516451:[()=>n.e(99128).then(n.bind(n,23312)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Overview.md",23312],"1587ba1d":[()=>n.e(43156).then(n.bind(n,86447)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Overview.md",86447],"15a59c33":[()=>n.e(57924).then(n.bind(n,6296)),"@site/versioned_docs/version-0.11.3/Deploy Models/Overview.md",6296],"15ea8379":[()=>n.e(19483).then(n.bind(n,71915)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",71915],"166d99f5":[()=>n.e(29696).then(n.bind(n,82914)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",82914],"16d05917":[()=>n.e(38759).then(n.bind(n,2727)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",2727],"16d8ca91":[()=>n.e(39402).then(n.bind(n,2422)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",2422],"1766722a":[()=>n.e(17817).then(n.bind(n,97592)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",97592],17896441:[()=>Promise.all([n.e(40532),n.e(24572),n.e(27918)]).then(n.bind(n,78945)),"@theme/DocItem",78945],"1789daeb":[()=>n.e(42377).then(n.bind(n,86544)),"@site/versioned_docs/version-1.0.2/Reference/Docker Setup.md",86544],"179d57fa":[()=>n.e(71580).then(n.bind(n,59171)),"@site/versioned_docs/version-1.0.1/Deploy Models/Quickstart - Deploying a Classifier.md",59171],"17f418b7":[()=>n.e(80837).then(n.bind(n,84998)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Install.md",84998],"1842239f":[()=>Promise.all([n.e(40532),n.e(14535)]).then(n.bind(n,3314)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_vw.md",3314],"18c7d2a3":[()=>n.e(15800).then(n.bind(n,62323)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",62323],"18c9c2ba":[()=>n.e(68585).then(n.bind(n,89326)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard.md",89326],"1a215693":[()=>n.e(71535).then(n.bind(n,85042)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",85042],"1a3608a9":[()=>n.e(24484).then(n.bind(n,75661)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",75661],"1a4e3797":[()=>Promise.all([n.e(40532),n.e(97920)]).then(n.bind(n,39172)),"@theme/SearchPage",39172],"1a62d1af":[()=>n.e(96960).then(n.bind(n,26408)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",26408],"1abf1e18":[()=>n.e(15343).then(n.bind(n,57819)),"@site/blog/2019-08-24-Welcome to Azure Cognitive Services.md",57819],"1ad91055":[()=>n.e(4193).then(n.bind(n,81719)),"@site/versioned_docs/version-1.0.1/Overview.md",81719],"1b3ab3b8":[()=>Promise.all([n.e(40532),n.e(13431)]).then(n.bind(n,53630)),"@site/docs/Quick Examples/transformers/transformers_cognitive.md",53630],"1b7af47d":[()=>n.e(14273).then(n.bind(n,51611)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",51611],"1be78505":[()=>Promise.all([n.e(40532),n.e(29514)]).then(n.bind(n,19963)),"@theme/DocPage",19963],"1d820955":[()=>n.e(81913).then(n.bind(n,58442)),"@site/docs/Explore Algorithms/LightGBM/Overview.md",58442],"1db1f785":[()=>n.e(40950).then(n.bind(n,96420)),"@site/docs/Reference/Developer Setup.md",96420],"1dc2f362":[()=>n.e(89864).then(n.bind(n,69454)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",69454],"1e0d2d3b":[()=>n.e(4503).then(n.bind(n,34824)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",34824],"1e14124f":[()=>n.e(96350).then(n.bind(n,35821)),"@site/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",35821],"1e57c92a":[()=>n.e(36958).then(n.bind(n,87204)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",87204],"1ef65056":[()=>Promise.all([n.e(40532),n.e(72326)]).then(n.bind(n,86903)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_vw.md",86903],"1f31a275":[()=>n.e(94353).then(n.bind(n,53512)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Install.md",53512],"1f6f5ac9":[()=>n.e(1123).then(n.bind(n,9198)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Getting Started.md",9198],20641120:[()=>n.e(787).then(n.bind(n,65432)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",65432],"20e0fe38":[()=>n.e(21102).then(n.bind(n,76411)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Autologging.md",76411],"2137a7cd":[()=>n.e(91391).then(n.bind(n,13295)),"@site/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",13295],"21f47424":[()=>Promise.all([n.e(40532),n.e(37474)]).then(n.bind(n,25305)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_vw.md",25305],"22444eb9":[()=>n.e(30335).then(n.bind(n,16807)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",16807],"22f921cc":[()=>n.e(35517).then(n.bind(n,18751)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",18751],"232d09b0":[()=>n.e(69314).then(n.bind(n,73850)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Getting Started.md",73850],"23af8e3e":[()=>n.e(82723).then(n.bind(n,98894)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Overview.md",98894],"2406587d":[()=>n.e(99119).then(n.bind(n,44519)),"@site/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",44519],"2461af4d":[()=>n.e(70880).then(n.bind(n,29727)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Langchain.md",29727],"255635fc":[()=>n.e(1172).then(n.bind(n,90421)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",90421],"25fb872c":[()=>n.e(48602).then(n.bind(n,502)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Autologging.md",502],26631026:[()=>n.e(78480).then(n.bind(n,22137)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",22137],"268b7b40":[()=>n.e(7234).then(n.bind(n,84922)),"@site/versioned_docs/version-1.0.2/Get Started/Quickstart - Your First Models.md",84922],"275c0efd":[()=>n.e(19950).then(n.bind(n,68567)),"@site/versioned_docs/version-1.0.3/Reference/Contributor Guide.md",68567],"27de0303":[()=>n.e(37334).then(n.bind(n,78849)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",78849],"283b8f99":[()=>n.e(95521).then(n.bind(n,51404)),"@site/docs/Get Started/Quickstart - Your First Models.md",51404],"294c060e":[()=>n.e(97165).then(n.bind(n,58142)),"@site/versioned_docs/version-1.0.4/Reference/Contributor Guide.md",58142],"295a8e78":[()=>n.e(26021).then(n.bind(n,67319)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",67319],"296054a7":[()=>n.e(66945).then(n.bind(n,87853)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",87853],"2a023663":[()=>n.e(97715).then(n.bind(n,23168)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Autologging.md",23168],"2b30bc14":[()=>n.e(19603).then(n.bind(n,12876)),"@site/blog/2019-08-24-Welcome to Azure Cognitive Services.md?truncated=true",12876],"2bcda99e":[()=>n.e(23941).then(n.bind(n,41368)),"@site/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",41368],"2c5f0abd":[()=>n.e(35015).then(n.bind(n,95677)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",95677],"2c98ca3e":[()=>n.e(62106).then(n.bind(n,6036)),"@site/docs/Deploy Models/Quickstart - Deploying a Classifier.md",6036],"2cec6bcf":[()=>n.e(41987).then(n.bind(n,95992)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Cyber ML.md",95992],"2d527871":[()=>n.e(8749).then(n.bind(n,90527)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",90527],"2d5b0e6a":[()=>n.e(28957).then(n.bind(n,33468)),"@site/blog/2018-04-01-Flexible and Scalable Deep Learning with MMLSpark.md?truncated=true",33468],"2d8a44d7":[()=>n.e(44534).then(n.bind(n,79259)),"@site/versioned_docs/version-1.0.3/Get Started/Install SynapseML.md",79259],"2e5f4263":[()=>n.e(52447).then(n.bind(n,18126)),"@site/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",18126],"2e93ae0f":[()=>n.e(40023).then(n.bind(n,60199)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",60199],"2ece0fbe":[()=>n.e(11561).then(n.bind(n,48438)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier.md",48438],"2fb6c1b0":[()=>n.e(80982).then(n.bind(n,76241)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",76241],"318ccb43":[()=>n.e(93377).then(n.bind(n,52979)),"@site/docs/Explore Algorithms/AI Services/Geospatial Services.md",52979],"31bb7274":[()=>n.e(67394).then(n.bind(n,42024)),"@site/docs/Explore Algorithms/Regression/Quickstart - Train Regressor.md",42024],"31ee0fa3":[()=>n.e(42428).then(n.bind(n,28431)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",28431],"3201258e":[()=>n.e(68526).then(n.bind(n,36362)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",36362],"321ba9c5":[()=>n.e(85332).then(n.bind(n,10360)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Geospatial Services.md",10360],"325e22bd":[()=>Promise.all([n.e(40532),n.e(92321)]).then(n.bind(n,23819)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_cognitive.md",23819],"327be84b":[()=>n.e(38399).then(n.bind(n,17630)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",17630],"328f3af4":[()=>n.e(42037).then(n.bind(n,16407)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/ONNX.md",16407],"32fe2e34":[()=>n.e(226).then(n.bind(n,28506)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/OpenAI.md",28506],"336404dc":[()=>n.e(43488).then(n.bind(n,49923)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Overview.md",49923],"33b3776b":[()=>n.e(5254).then(n.bind(n,35734)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",35734],"345903d4":[()=>n.e(52433).then(n.bind(n,31889)),"@site/versioned_docs/version-1.0.1/Reference/Docker Setup.md",31889],"34f00221":[()=>Promise.all([n.e(40532),n.e(37853),n.e(76454)]).then(n.bind(n,63785)),"@site/src/pages/videos.js",63785],"34ff21b6":[()=>Promise.all([n.e(40532),n.e(79362)]).then(n.bind(n,93488)),"@site/docs/Quick Examples/estimators/estimators_cognitive.md",93488],"35826f14":[()=>n.e(23894).then(n.bind(n,88680)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Install.md",88680],"35bdc661":[()=>n.e(27813).then(n.bind(n,58782)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",58782],"3647ac9b":[()=>n.e(92982).then(n.bind(n,66377)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",66377],"368de161":[()=>n.e(1985).then(n.bind(n,83832)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Train Classifier.md",83832],"37ab9beb":[()=>n.e(64992).then(n.bind(n,1050)),"@site/versioned_docs/version-0.11.3/Deploy Models/Quickstart - Deploying a Classifier.md",1050],"38b375cc":[()=>n.e(72249).then(n.bind(n,86440)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",86440],"38d6824e":[()=>n.e(25990).then(n.bind(n,85202)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",85202],"39122aab":[()=>n.e(57689).then(n.bind(n,37155)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",37155],"391cb159":[()=>n.e(41867).then(n.bind(n,55482)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/LightGBM/Overview.md",55482],"39582d99":[()=>Promise.all([n.e(40532),n.e(77697)]).then(n.bind(n,72711)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_opencv.md",72711],"39a50b22":[()=>n.e(29845).then(n.bind(n,13442)),"@site/versioned_docs/version-1.0.3/Overview.md",13442],"3a1147b5":[()=>n.e(82668).then(n.bind(n,92857)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers.md",92857],"3a4f9d93":[()=>n.e(15487).then(n.bind(n,90849)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Getting Started.md",90849],"3a7a88af":[()=>n.e(49814).then(n.bind(n,31377)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",31377],"3ac37a06":[()=>Promise.all([n.e(40532),n.e(91103)]).then(n.bind(n,98105)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_deep_learning.md",98105],"3b5de274":[()=>n.e(11977).then(n.bind(n,69625)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/ONNX.md",69625],"3b75a1c6":[()=>n.e(78047).then(n.bind(n,18615)),"@site/versioned_docs/version-1.0.4/Deploy Models/Overview.md",18615],"3bbe0f34":[()=>n.e(64172).then(n.bind(n,90054)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Image Explainers.md",90054],"3bed3e2f":[()=>n.e(58846).then(n.bind(n,38189)),"@site/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",38189],"3c1f4383":[()=>n.e(16551).then(n.bind(n,52839)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Install.md",52839],"3c22aa66":[()=>n.e(86482).then(n.bind(n,18590)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",18590],"3cbfbf75":[()=>n.e(1773).then(n.bind(n,97312)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor.md",97312],"3d963cd5":[()=>n.e(72286).then(n.bind(n,62991)),"@site/versioned_docs/version-1.0.4/Deploy Models/Quickstart - Deploying a Classifier.md",62991],"3dafa002":[()=>n.e(76258).then(n.bind(n,35440)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",35440],"3e1e84da":[()=>n.e(34397).then(n.bind(n,69813)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/OpenAI.md",69813],"3e7c5569":[()=>n.e(49778).then(n.bind(n,49138)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",49138],"3e9a7422":[()=>Promise.all([n.e(40532),n.e(83393)]).then(n.bind(n,73297)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_vw.md",73297],"3ea19ec0":[()=>n.e(17026).then(n.bind(n,90308)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",90308],"3ee83f58":[()=>n.e(42802).then(n.bind(n,46489)),"@site/versioned_docs/version-1.0.3/Reference/Docker Setup.md",46489],"3ef20ed6":[()=>n.e(61407).then(n.bind(n,2784)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",2784],"3fb29942":[()=>n.e(69559).then(n.t.bind(n,24469,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-blog/default/plugin-route-context-module-100.json",24469],"403e642d":[()=>n.e(60055).then(n.bind(n,5312)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",5312],"405fabd0":[()=>n.e(5253).then(n.bind(n,3354)),"@site/versioned_docs/version-1.0.4/Reference/Docker Setup.md",3354],"408524d4":[()=>Promise.all([n.e(40532),n.e(48007)]).then(n.bind(n,57898)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_cognitive.md",57898],"40b49758":[()=>n.e(37069).then(n.bind(n,33548)),"@site/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",33548],"4130db01":[()=>n.e(86184).then(n.bind(n,40606)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",40606],"41ee0cff":[()=>n.e(55702).then(n.bind(n,5380)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",5380],"41f327fc":[()=>n.e(8966).then(n.bind(n,81631)),"@site/docs/Explore Algorithms/Classification/Quickstart - Train Classifier.md",81631],"421d486c":[()=>Promise.all([n.e(40532),n.e(1084)]).then(n.bind(n,24310)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_core.md",24310],"423540f2":[()=>n.e(42852).then(n.bind(n,80918)),"@site/versioned_docs/version-1.0.1/Get Started/Create a Spark Cluster.md",80918],"42736d5f":[()=>n.e(42947).then(n.bind(n,25348)),"@site/docs/Explore Algorithms/Deep Learning/Getting Started.md",25348],"445012ce":[()=>n.e(3871).then(n.bind(n,56324)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Overview.md",56324],"445a1b28":[()=>Promise.all([n.e(40532),n.e(18373)]).then(n.bind(n,73225)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_lightgbm.md",73225],"44ff0d07":[()=>n.e(65437).then(n.bind(n,62634)),"@site/docs/Reference/R Setup.md",62634],"455aa1bd":[()=>n.e(43251).then(n.bind(n,39068)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Text Explainers.md",39068],"4585b70e":[()=>Promise.all([n.e(40532),n.e(90544)]).then(n.bind(n,53232)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_deep_learning.md",53232],"4631a032":[()=>n.e(37676).then(n.bind(n,25217)),"@site/versioned_docs/version-1.0.4/Reference/Quickstart - LightGBM in Dotnet.md",25217],"47bbce76":[()=>n.e(30464).then(n.bind(n,14008)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",14008],"4801d370":[()=>n.e(59394).then(n.bind(n,58406)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",58406],"48329b94":[()=>Promise.all([n.e(40532),n.e(23979)]).then(n.bind(n,59100)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_deep_learning.md",59100],"48cf7354":[()=>n.e(82994).then(n.bind(n,82516)),"@site/docs/Explore Algorithms/Responsible AI/Tabular Explainers.md",82516],"494501bd":[()=>Promise.all([n.e(40532),n.e(43426)]).then(n.bind(n,41878)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_core.md",41878],"4997ef4f":[()=>n.e(74772).then(n.bind(n,77565)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",77565],"49e6864b":[()=>n.e(22707).then(n.bind(n,52565)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Image Explainers.md",52565],"4b18e5ae":[()=>n.e(38962).then(n.bind(n,16208)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",16208],"4bbbdfcf":[()=>n.e(6413).then(n.bind(n,87267)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",87267],"4bea8531":[()=>n.e(12840).then(n.bind(n,49218)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",49218],"4cee39d7":[()=>n.e(94756).then(n.bind(n,74001)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",74001],"4d31bfb3":[()=>n.e(58092).then(n.bind(n,66388)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",66388],"4db82253":[()=>n.e(97381).then(n.bind(n,45355)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",45355],"4de42975":[()=>n.e(91570).then(n.bind(n,68762)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",68762],"4e044b4e":[()=>Promise.all([n.e(40532),n.e(85847)]).then(n.bind(n,40114)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_lightgbm.md",40114],"4e105cba":[()=>n.e(39596).then(n.bind(n,49164)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",49164],"4e3910af":[()=>n.e(23544).then(n.bind(n,3030)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Cyber ML.md",3030],"4e432063":[()=>n.e(97988).then(n.t.bind(n,43260,19)),"~docs/default/version-0-11-4-metadata-prop-456.json",43260],"4ef2f7cf":[()=>n.e(62675).then(n.bind(n,82055)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",82055],"4f1ca6a9":[()=>n.e(9540).then(n.bind(n,69671)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Overview.md",69671],"4f81391a":[()=>n.e(15165).then(n.bind(n,91769)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",91769],"5066efb2":[()=>n.e(88003).then(n.bind(n,96933)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",96933],"50af03e4":[()=>n.e(36204).then(n.bind(n,79410)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Text Explainers.md",79410],"50f80512":[()=>n.e(77682).then(n.bind(n,90256)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",90256],"513002de":[()=>n.e(1802).then(n.bind(n,76933)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",76933],"513214d5":[()=>Promise.all([n.e(40532),n.e(3701)]).then(n.bind(n,98157)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_opencv.md",98157],"514f485e":[()=>n.e(18828).then(n.bind(n,53701)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",53701],"51878b77":[()=>n.e(77225).then(n.bind(n,6382)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",6382],"51dbb02f":[()=>n.e(16975).then(n.bind(n,11091)),"@site/docs/Explore Algorithms/Responsible AI/Explanation Dashboard.md",11091],"521f9727":[()=>Promise.all([n.e(40532),n.e(31333)]).then(n.bind(n,66202)),"@site/docs/Quick Examples/estimators/estimators_causal.md",66202],"5225b7e0":[()=>n.e(73249).then(n.bind(n,25101)),"@site/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",25101],"52880d18":[()=>n.e(62750).then(n.bind(n,98124)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",98124],"53a5cb1e":[()=>n.e(94548).then(n.bind(n,51847)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor.md",51847],"5407376f":[()=>n.e(22287).then(n.bind(n,87405)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",87405],"54287b47":[()=>n.e(35328).then(n.bind(n,81589)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",81589],"548964ce":[()=>Promise.all([n.e(40532),n.e(19465)]).then(n.bind(n,25947)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_vw.md",25947],55203621:[()=>n.e(3554).then(n.bind(n,40218)),"@site/versioned_docs/version-1.0.2/Use with MLFlow/Overview.md",40218],"556ad472":[()=>n.e(80751).then(n.bind(n,43170)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",43170],"56087ddb":[()=>n.e(72293).then(n.bind(n,25952)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",25952],"561bd03d":[()=>n.e(32185).then(n.bind(n,97308)),"@site/blog/2020-12-01-Large-Scale Intelligent Microservices.md",97308],"56543a15":[()=>n.e(63402).then(n.bind(n,26786)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",26786],"567cd2fd":[()=>n.e(7201).then(n.bind(n,4423)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Image Explainers.md",4423],"56d039a1":[()=>Promise.all([n.e(40532),n.e(85729)]).then(n.bind(n,48482)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_cognitive.md",48482],"57e687e8":[()=>n.e(69159).then(n.bind(n,95823)),"@site/versioned_docs/version-0.11.3/Reference/Docker Setup.md",95823],"57ef17c5":[()=>n.e(95602).then(n.bind(n,82569)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",82569],"58486a98":[()=>Promise.all([n.e(40532),n.e(76572)]).then(n.bind(n,38697)),"@site/versioned_docs/version-1.0.4/Reference/Dotnet Setup.md",38697],"5887dc62":[()=>Promise.all([n.e(40532),n.e(77969)]).then(n.bind(n,89912)),"@site/versioned_docs/version-1.0.1/Reference/Dotnet Setup.md",89912],"589adaf2":[()=>Promise.all([n.e(40532),n.e(36572)]).then(n.bind(n,41965)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_causal.md",41965],"58b8f176":[()=>n.e(99374).then(n.bind(n,58838)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences.md",58838],"5962ef6e":[()=>Promise.all([n.e(40532),n.e(12630)]).then(n.bind(n,27505)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_core.md",27505],"59d9fecc":[()=>n.e(84908).then(n.bind(n,53217)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Causal Inference/Overview.md",53217],"59e4d88e":[()=>n.e(2381).then(n.bind(n,60701)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/ONNX.md",60701],"59ef8022":[()=>n.e(48491).then(n.bind(n,23796)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis.md",23796],"5a3e1bca":[()=>n.e(94702).then(n.bind(n,83285)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",83285],"5b779334":[()=>n.e(16833).then(n.bind(n,88864)),"@site/docs/Reference/Quickstart - LightGBM in Dotnet.md",88864],"5bf420e9":[()=>Promise.all([n.e(40532),n.e(32855)]).then(n.bind(n,6805)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_core.md",6805],"5c0b440a":[()=>Promise.all([n.e(40532),n.e(50783)]).then(n.bind(n,2885)),"@site/versioned_docs/version-1.0.4/Quick Examples/transformers/transformers_deep_learning.md",2885],"5c18deb5":[()=>n.e(33358).then(n.bind(n,24964)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Geospatial Services.md",24964],"5cfa133c":[()=>n.e(62079).then(n.bind(n,80697)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",80697],"5dbf4619":[()=>n.e(92867).then(n.bind(n,76669)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis.md",76669],"5e008bd1":[()=>n.e(6304).then(n.bind(n,20671)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",20671],"5e7c3303":[()=>n.e(38781).then(n.bind(n,72613)),"@site/docs/Explore Algorithms/Deep Learning/Distributed Training.md",72613],"5e9f5e1a":[()=>Promise.resolve().then(n.bind(n,36809)),"@generated/docusaurus.config",36809],"5eddefdd":[()=>n.e(30013).then(n.bind(n,61820)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",61820],"5ef28062":[()=>n.e(36746).then(n.bind(n,30356)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",30356],"5efd5006":[()=>n.e(37742).then(n.bind(n,41783)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",41783],"5f17911b":[()=>n.e(32599).then(n.bind(n,93084)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",93084],"603706f6":[()=>Promise.all([n.e(40532),n.e(15149)]).then(n.bind(n,43006)),"@site/versioned_docs/version-0.11.3/Reference/Dotnet Setup.md",43006],"604b2d50":[()=>Promise.all([n.e(40532),n.e(83790)]).then(n.bind(n,29364)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_vw.md",29364],"6094981e":[()=>n.e(84298).then(n.bind(n,78053)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",78053],"60a2189a":[()=>n.e(3953).then(n.bind(n,82469)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis.md",82469],"610d430a":[()=>n.e(40659).then(n.bind(n,33728)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Overview.md",33728],"627aefca":[()=>n.e(30301).then(n.t.bind(n,15745,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-pages/default/plugin-route-context-module-100.json",15745],"62983deb":[()=>n.e(76843).then(n.bind(n,19518)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Langchain.md",19518],"62ccfbde":[()=>n.e(82681).then(n.bind(n,70375)),"@site/versioned_docs/version-1.0.4/Get Started/Install SynapseML.md",70375],"63cd8b22":[()=>n.e(17973).then(n.bind(n,47831)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",47831],"644ff257":[()=>n.e(51910).then(n.bind(n,97780)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Text Explainers.md",97780],"645abc7d":[()=>n.e(31836).then(n.bind(n,32321)),"@site/docs/Reference/Docker Setup.md",32321],"66d63bfc":[()=>n.e(17542).then(n.t.bind(n,84370,19)),"~docs/default/version-1-0-3-metadata-prop-7b4.json",84370],"66fcb786":[()=>n.e(87597).then(n.bind(n,69554)),"@site/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",69554],"67e06a8b":[()=>n.e(86704).then(n.bind(n,11969)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",11969],"69bb4b11":[()=>Promise.all([n.e(40532),n.e(90288)]).then(n.bind(n,14233)),"@site/docs/Quick Examples/transformers/transformers_opencv.md",14233],"69dc6d0a":[()=>n.e(99161).then(n.bind(n,94325)),"@site/docs/Explore Algorithms/Responsible AI/Data Balance Analysis.md",94325],"6a07b55e":[()=>n.e(70406).then(n.bind(n,44243)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",44243],"6a26e359":[()=>n.e(33195).then(n.bind(n,35071)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",35071],"6a95f87e":[()=>n.e(73796).then(n.t.bind(n,56800,19)),"~blog/default/synapse-ml-blog-963.json",56800],"6a9ad78d":[()=>Promise.all([n.e(40532),n.e(51317)]).then(n.bind(n,5599)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_causal.md",5599],"6b40ec54":[()=>n.e(22712).then(n.bind(n,36338)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",36338],"6b9bdd6f":[()=>n.e(51105).then(n.bind(n,60038)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Geospatial Services.md",60038],"6bdbf6b8":[()=>n.e(98583).then(n.bind(n,351)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/LightGBM/Overview.md",351],"6c4c8509":[()=>n.e(42522).then(n.bind(n,25434)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",25434],"6c61f93b":[()=>Promise.all([n.e(40532),n.e(3202)]).then(n.bind(n,78968)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_causal.md",78968],"6c7ccb6d":[()=>n.e(12398).then(n.bind(n,18527)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",18527],"6c7d288d":[()=>n.e(36224).then(n.bind(n,70515)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",70515],"6c82e982":[()=>n.e(2070).then(n.bind(n,47636)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",47636],"6e159789":[()=>n.e(3611).then(n.bind(n,6930)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",6930],"6e2678b7":[()=>n.e(28465).then(n.bind(n,31750)),"@site/versioned_docs/version-1.0.3/Deploy Models/Quickstart - Deploying a Classifier.md",31750],"6e4e5cd9":[()=>n.e(21401).then(n.bind(n,39225)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",39225],"6e71bda3":[()=>n.e(28955).then(n.bind(n,63355)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/ONNX.md",63355],"6eddee4c":[()=>n.e(17295).then(n.bind(n,82586)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",82586],"6f479459":[()=>n.e(74963).then(n.bind(n,33261)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Image Explainers.md",33261],"6f8cd013":[()=>n.e(38078).then(n.bind(n,73529)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",73529],"6ff7775f":[()=>n.e(89461).then(n.bind(n,48404)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",48404],"707d2a35":[()=>n.e(6633).then(n.bind(n,38429)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine.md",38429],"70b306ba":[()=>n.e(38377).then(n.bind(n,61519)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/ONNX.md",61519],"70e8008e":[()=>n.e(88838).then(n.bind(n,12521)),"@site/versioned_docs/version-1.0.4/Use with MLFlow/Install.md",12521],"70fe7dd7":[()=>n.e(44178).then(n.bind(n,77849)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Overview.md",77849],71042800:[()=>n.e(59484).then(n.bind(n,30382)),"@site/versioned_docs/version-0.11.3/Reference/Quickstart - LightGBM in Dotnet.md",30382],"7275f11a":[()=>n.e(34940).then(n.bind(n,55516)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",55516],"7296bed0":[()=>n.e(98528).then(n.bind(n,97421)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",97421],"73c2022c":[()=>n.e(847).then(n.bind(n,32422)),"@site/docs/Get Started/Install SynapseML.md",32422],"73cf52e8":[()=>n.e(4026).then(n.bind(n,88585)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",88585],"74091a62":[()=>n.e(63196).then(n.bind(n,89086)),"@site/versioned_docs/version-1.0.2/Reference/R Setup.md",89086],"74659d33":[()=>n.e(66167).then(n.bind(n,82605)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine.md",82605],75406112:[()=>n.e(73060).then(n.bind(n,73375)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",73375],"75514f6a":[()=>n.e(12824).then(n.bind(n,95694)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor.md",95694],75801678:[()=>Promise.all([n.e(40532),n.e(6289)]).then(n.bind(n,62602)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_core.md",62602],"7649ec47":[()=>n.e(45370).then(n.bind(n,34329)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",34329],"767a7177":[()=>n.e(35040).then(n.bind(n,49352)),"@site/versioned_docs/version-0.11.4/Get Started/Set up Cognitive Services.md",49352],"76b137bf":[()=>n.e(55276).then(n.bind(n,70993)),"@site/versioned_docs/version-1.0.1/Reference/Contributor Guide.md",70993],"76bf568a":[()=>n.e(59823).then(n.t.bind(n,38484,19)),"~docs/default/version-1-0-4-metadata-prop-029.json",38484],"76d23901":[()=>n.e(16269).then(n.bind(n,46954)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",46954],"77c259dd":[()=>n.e(70224).then(n.bind(n,60878)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Overview.md",60878],"78aebd5d":[()=>n.e(91895).then(n.bind(n,38547)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",38547],"78cb5959":[()=>n.e(5909).then(n.bind(n,87957)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",87957],"79114dfe":[()=>n.e(41944).then(n.bind(n,92175)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers.md",92175],"792eae7e":[()=>Promise.all([n.e(40532),n.e(74275)]).then(n.bind(n,48117)),"@site/docs/Quick Examples/estimators/estimators_lightgbm.md",48117],"79d79762":[()=>Promise.all([n.e(40532),n.e(46845)]).then(n.bind(n,86931)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_cognitive.md",86931],"7a1d9beb":[()=>n.e(75088).then(n.bind(n,17021)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard.md",17021],"7aa49360":[()=>n.e(99147).then(n.bind(n,82379)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Overview.md",82379],"7aa7b8e0":[()=>Promise.all([n.e(40532),n.e(70344)]).then(n.bind(n,31036)),"@site/versioned_docs/version-0.11.4/Reference/Dotnet Setup.md",31036],"7ad384dc":[()=>n.e(45589).then(n.bind(n,60946)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/LightGBM/Overview.md",60946],"7aeb6f16":[()=>n.e(24046).then(n.bind(n,99342)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences.md",99342],"7b60d8e3":[()=>n.e(403).then(n.bind(n,47989)),"@site/versioned_docs/version-1.0.2/Reference/Contributor Guide.md",47989],"7bf16cc6":[()=>n.e(86248).then(n.bind(n,75144)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",75144],"7c007ec6":[()=>n.e(29283).then(n.bind(n,65302)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",65302],"7c2e1100":[()=>n.e(58861).then(n.bind(n,32961)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Langchain.md",32961],"80472ecc":[()=>n.e(91078).then(n.bind(n,43253)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis.md",43253],"814f3328":[()=>n.e(52535).then(n.t.bind(n,45641,19)),"~blog/default/blog-post-list-prop-default.json",45641],"8181e18a":[()=>n.e(69226).then(n.bind(n,15489)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Getting Started.md",15489],"82878c84":[()=>n.e(94916).then(n.bind(n,81962)),"@site/versioned_docs/version-0.11.3/Get Started/Create a Spark Cluster.md",81962],"82d7cc79":[()=>n.e(96816).then(n.bind(n,68560)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",68560],"83203dd1":[()=>n.e(89450).then(n.bind(n,8346)),"@site/versioned_docs/version-1.0.3/Get Started/Set up Cognitive Services.md",8346],"8328d740":[()=>n.e(91921).then(n.bind(n,18567)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers.md",18567],"83586bff":[()=>n.e(37001).then(n.bind(n,22760)),"@site/docs/Explore Algorithms/OpenCV/Image Transformations.md",22760],"8462374b":[()=>Promise.all([n.e(40532),n.e(66808)]).then(n.bind(n,14707)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_core.md",14707],"8590feaa":[()=>Promise.all([n.e(40532),n.e(91871)]).then(n.bind(n,53505)),"@site/docs/Quick Examples/transformers/transformers_vw.md",53505],"85aed2f9":[()=>n.e(91747).then(n.bind(n,31029)),"@site/versioned_docs/version-0.11.4/Reference/R Setup.md",31029],"866b91a1":[()=>n.e(83781).then(n.bind(n,23442)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Text Explainers.md",23442],"86ed1ff2":[()=>n.e(53090).then(n.bind(n,9814)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",9814],"8799ef55":[()=>n.e(45938).then(n.bind(n,13835)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",13835],"888e2229":[()=>n.e(67448).then(n.bind(n,9850)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",9850],"88b63415":[()=>n.e(360).then(n.bind(n,57933)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",57933],"88e44e99":[()=>Promise.all([n.e(40532),n.e(78817)]).then(n.bind(n,14859)),"@site/docs/Quick Examples/estimators/estimators_vw.md",14859],"88f8e3e1":[()=>n.e(22970).then(n.bind(n,69278)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenCV/Image Transformations.md",69278],"8a1bca9d":[()=>n.e(26871).then(n.bind(n,44359)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Overview.md",44359],"8a46fce0":[()=>n.e(92836).then(n.bind(n,59438)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",59438],"8af72580":[()=>n.e(22577).then(n.bind(n,84586)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Overview.md",84586],"8bff2444":[()=>n.e(19336).then(n.bind(n,17465)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Overview.md",17465],"8cb11643":[()=>n.e(95974).then(n.bind(n,8169)),"@site/blog/2019-10-02-MMLSpark empowering AI for Good with Mark Hamilton.md?truncated=true",8169],"8d990105":[()=>n.e(28456).then(n.bind(n,94078)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",94078],"8e32e44d":[()=>n.e(73419).then(n.bind(n,21554)),"@site/versioned_docs/version-0.11.3/Reference/Developer Setup.md",21554],"8eb438b4":[()=>n.e(85043).then(n.bind(n,34540)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",34540],"8fd0c721":[()=>n.e(80112).then(n.bind(n,99640)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Overview.md",99640],"8ff6772d":[()=>n.e(32777).then(n.bind(n,30213)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Tabular Explainers.md",30213],"900b935c":[()=>n.e(65942).then(n.bind(n,69180)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",69180],"905ce8b5":[()=>n.e(29535).then(n.bind(n,34527)),"@site/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",34527],90742975:[()=>n.e(36005).then(n.bind(n,41454)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis.md",41454],"90e4432b":[()=>n.e(15095).then(n.bind(n,40814)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",40814],"90f00051":[()=>n.e(33631).then(n.bind(n,2040)),"@site/versioned_docs/version-1.0.1/Get Started/Quickstart - Your First Models.md",2040],"91e9cb67":[()=>n.e(73443).then(n.bind(n,71766)),"@site/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",71766],"92ce4fd5":[()=>n.e(23838).then(n.bind(n,61997)),"@site/versioned_docs/version-1.0.2/Get Started/Set up Cognitive Services.md",61997],"935f2afb":[()=>n.e(80053).then(n.t.bind(n,1109,19)),"~docs/default/version-current-metadata-prop-751.json",1109],"93ba15e0":[()=>n.e(87452).then(n.bind(n,29850)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",29850],"943afd32":[()=>n.e(53050).then(n.bind(n,30322)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",30322],"94d743d6":[()=>n.e(94579).then(n.bind(n,12815)),"@site/docs/Deploy Models/Overview.md",12815],"9530a2cf":[()=>n.e(7509).then(n.bind(n,70325)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",70325],"95d9e891":[()=>n.e(63099).then(n.bind(n,6022)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",6022],"965ed185":[()=>Promise.all([n.e(40532),n.e(40852)]).then(n.bind(n,29009)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_cognitive.md",29009],"9688a5be":[()=>Promise.all([n.e(40532),n.e(51918)]).then(n.bind(n,52915)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_cognitive.md",52915],"96cf5ff0":[()=>n.e(46135).then(n.bind(n,35761)),"@site/docs/Explore Algorithms/OpenAI/OpenAI.md",35761],"97592aac":[()=>n.e(50653).then(n.bind(n,36490)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",36490],"9783ff24":[()=>Promise.all([n.e(40532),n.e(29006)]).then(n.bind(n,89448)),"@site/docs/Quick Examples/estimators/estimators_core.md",89448],"980b00ea":[()=>n.e(2901).then(n.bind(n,37673)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard.md",37673],"98258c5e":[()=>n.e(46559).then(n.bind(n,39475)),"@site/versioned_docs/version-1.0.2/Get Started/Install SynapseML.md",39475],"990f07de":[()=>n.e(58755).then(n.bind(n,27471)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",27471],"993c5022":[()=>n.e(42815).then(n.bind(n,16087)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/LightGBM/Overview.md",16087],"995576e9":[()=>n.e(43344).then(n.bind(n,61433)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",61433],"99e5ffa1":[()=>n.e(44504).then(n.bind(n,22327)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Getting Started.md",22327],"99f8fee5":[()=>n.e(67287).then(n.bind(n,1288)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",1288],"9b020c4a":[()=>n.e(52752).then(n.bind(n,69975)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",69975],"9c279ae7":[()=>n.e(50196).then(n.bind(n,91257)),"@site/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",91257],"9c5088a2":[()=>n.e(7925).then(n.bind(n,22299)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Autologging.md",22299],"9cb6fa7c":[()=>n.e(76515).then(n.bind(n,1458)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Causal Inference/Overview.md",1458],"9d095dba":[()=>Promise.all([n.e(40532),n.e(57231)]).then(n.bind(n,53620)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_core.md",53620],"9da9112f":[()=>n.e(8006).then(n.bind(n,61363)),"@site/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",61363],"9e4087bc":[()=>n.e(53608).then(n.bind(n,63169)),"@theme/BlogArchivePage",63169],"9e5b788f":[()=>n.e(40412).then(n.bind(n,71393)),"@site/versioned_docs/version-1.0.2/Reference/Developer Setup.md",71393],"9fa7e8eb":[()=>n.e(31896).then(n.bind(n,34021)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Cyber ML.md",34021],a0434473:[()=>n.e(79328).then(n.bind(n,401)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Langchain.md",401],a07d09c1:[()=>n.e(27572).then(n.bind(n,82679)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",82679],a0d8a1a2:[()=>n.e(61778).then(n.bind(n,83269)),"@site/versioned_docs/version-1.0.2/Get Started/Create a Spark Cluster.md",83269],a1824f78:[()=>n.e(68070).then(n.bind(n,55261)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",55261],a19ce767:[()=>n.e(89463).then(n.bind(n,32152)),"@site/docs/Get Started/Create a Spark Cluster.md",32152],a2b5e5df:[()=>n.e(89220).then(n.bind(n,96797)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenCV/Image Transformations.md",96797],a34b09d4:[()=>n.e(95768).then(n.bind(n,25401)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine.md",25401],a34c49d0:[()=>n.e(22007).then(n.bind(n,96633)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",96633],a3fd5a6d:[()=>Promise.all([n.e(40532),n.e(52730)]).then(n.bind(n,73778)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_opencv.md",73778],a406e231:[()=>n.e(37486).then(n.bind(n,30640)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",30640],a563fa93:[()=>n.e(8839).then(n.bind(n,82684)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",82684],a5ffe972:[()=>n.e(14341).then(n.bind(n,34581)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",34581],a608660b:[()=>n.e(67597).then(n.bind(n,90970)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",90970],a6191053:[()=>n.e(30927).then(n.bind(n,94736)),"@site/docs/Use with MLFlow/Overview.md",94736],a6aa9e1f:[()=>Promise.all([n.e(40532),n.e(24572),n.e(46048),n.e(93089)]).then(n.bind(n,93269)),"@theme/BlogListPage",93269],a81cf8b0:[()=>n.e(50040).then(n.bind(n,18943)),"@site/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",18943],a8456d9f:[()=>n.e(53572).then(n.bind(n,61158)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",61158],a877f9e5:[()=>Promise.all([n.e(40532),n.e(72141)]).then(n.bind(n,69956)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_vw.md",69956],a9330763:[()=>n.e(15511).then(n.bind(n,36077)),"@site/versioned_docs/version-0.11.4/Deploy Models/Overview.md",36077],aa5548ab:[()=>n.e(22341).then(n.bind(n,73108)),"@site/versioned_docs/version-1.0.1/Deploy Models/Overview.md",73108],ab9a8589:[()=>n.e(80707).then(n.bind(n,61167)),"@site/blog/2019-06-01-MMLSpark Unifying Machine Learning Ecosystems at Massive Scales.md?truncated=true",61167],aba932dd:[()=>n.e(36698).then(n.bind(n,17798)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",17798],ac39f7dc:[()=>n.e(15404).then(n.bind(n,4200)),"@site/versioned_docs/version-1.0.3/Get Started/Quickstart - Your First Models.md",4200],ac581902:[()=>n.e(10978).then(n.bind(n,25386)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",25386],ad10988d:[()=>Promise.all([n.e(40532),n.e(1854)]).then(n.bind(n,36305)),"@site/versioned_docs/version-0.11.3/Quick Examples/transformers/transformers_opencv.md",36305],ad139e3c:[()=>n.e(17538).then(n.bind(n,25612)),"@site/versioned_docs/version-0.11.4/Deploy Models/Quickstart - Deploying a Classifier.md",25612],ae89cd3b:[()=>n.e(39139).then(n.bind(n,46405)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",46405],af1e70b6:[()=>n.e(73602).then(n.bind(n,49816)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor.md",49816],affaa541:[()=>n.e(36673).then(n.bind(n,53727)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Train Regressor.md",53727],b066233f:[()=>n.e(28696).then(n.bind(n,37753)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",37753],b0e28723:[()=>n.e(36285).then(n.bind(n,10431)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",10431],b1706689:[()=>n.e(17655).then(n.bind(n,5358)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",5358],b28fe732:[()=>Promise.all([n.e(40532),n.e(87808)]).then(n.bind(n,99485)),"@site/versioned_docs/version-1.0.4/Quick Examples/estimators/estimators_lightgbm.md",99485],b2be3901:[()=>n.e(19775).then(n.bind(n,83907)),"@site/docs/Get Started/Set up Cognitive Services.md",83907],b3272230:[()=>n.e(806).then(n.bind(n,90766)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",90766],b328a361:[()=>n.e(17782).then(n.bind(n,92522)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",92522],b33598ac:[()=>n.e(92059).then(n.bind(n,13052)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Responsible AI/Image Explainers.md",13052],b35d004b:[()=>n.e(87384).then(n.bind(n,97178)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",97178],b35e9998:[()=>n.e(77390).then(n.bind(n,94975)),"@site/versioned_docs/version-1.0.3/Use with MLFlow/Autologging.md",94975],b38eec89:[()=>Promise.all([n.e(40532),n.e(87684)]).then(n.bind(n,97256)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_deep_learning.md",97256],b41f295c:[()=>n.e(3814).then(n.bind(n,17551)),"@site/versioned_docs/version-0.11.3/Reference/Contributor Guide.md",17551],b4898d44:[()=>n.e(46613).then(n.bind(n,30178)),"@site/versioned_docs/version-1.0.2/Overview.md",30178],b4a95996:[()=>n.e(51478).then(n.bind(n,1973)),"@site/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",1973],b4ae214b:[()=>n.e(36773).then(n.bind(n,80609)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",80609],b5375b6f:[()=>n.e(61428).then(n.bind(n,7902)),"@site/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",7902],b576c645:[()=>n.e(80322).then(n.bind(n,39461)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers.md",39461],b66ab6b5:[()=>n.e(688).then(n.bind(n,37483)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",37483],b66e2100:[()=>n.e(41719).then(n.bind(n,16930)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Distributed Training.md",16930],b7226801:[()=>n.e(36585).then(n.bind(n,24646)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",24646],b72abe57:[()=>n.e(2195).then(n.bind(n,83847)),"@site/blog/2020-12-01-Large-Scale Intelligent Microservices.md?truncated=true",83847],b739c60a:[()=>n.e(30023).then(n.bind(n,49580)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",49580],b75118f0:[()=>n.e(61442).then(n.bind(n,40674)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",40674],b7787e72:[()=>n.e(15253).then(n.bind(n,23339)),"@site/versioned_docs/version-1.0.1/Reference/R Setup.md",23339],b7802ae1:[()=>n.e(38859).then(n.bind(n,49665)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",49665],b7c6295c:[()=>n.e(64233).then(n.bind(n,71955)),"@site/versioned_docs/version-1.0.1/Reference/Developer Setup.md",71955],b80b493a:[()=>n.e(86856).then(n.bind(n,92934)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks.md",92934],b83021be:[()=>n.e(587).then(n.bind(n,6283)),"@site/docs/Explore Algorithms/Causal Inference/Overview.md",6283],b88f1f9b:[()=>n.e(53338).then(n.bind(n,92235)),"@site/versioned_docs/version-1.0.4/Reference/Developer Setup.md",92235],b8963dc0:[()=>n.e(5287).then(n.bind(n,65710)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",65710],b92ffdc6:[()=>n.e(80874).then(n.bind(n,29864)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",29864],b940e0b5:[()=>n.e(58738).then(n.bind(n,7662)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",7662],b962e007:[()=>n.e(24594).then(n.bind(n,46495)),"@site/blog/2019-10-02-MMLSpark empowering AI for Good with Mark Hamilton.md",46495],b9f125cc:[()=>n.e(29806).then(n.bind(n,43252)),"@site/versioned_docs/version-0.11.4/Overview.md",43252],ba9c0924:[()=>n.e(67218).then(n.bind(n,66174)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",66174],bb7a48da:[()=>Promise.all([n.e(40532),n.e(39242)]).then(n.bind(n,52528)),"@site/docs/Quick Examples/transformers/transformers_core.md",52528],bc4bf151:[()=>n.e(91755).then(n.bind(n,7297)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference.md",7297],bc6a9944:[()=>n.e(98462).then(n.bind(n,38334)),"@site/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",38334],bc85b804:[()=>Promise.all([n.e(40532),n.e(45503)]).then(n.bind(n,20408)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_lightgbm.md",20408],bd1bcb2c:[()=>n.e(95584).then(n.bind(n,66193)),"@site/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",66193],be95fd8f:[()=>n.e(57887).then(n.bind(n,8801)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML.md",8801],bece3771:[()=>n.e(55538).then(n.bind(n,57450)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data.md",57450],bef0d2d3:[()=>n.e(16286).then(n.bind(n,3766)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Cyber ML.md",3766],bf002efe:[()=>n.e(53017).then(n.bind(n,1347)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",1347],bf04857e:[()=>n.e(38846).then(n.bind(n,76282)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Data Cleaning.md",76282],bf8a5c8a:[()=>n.e(63002).then(n.bind(n,42365)),"@site/versioned_docs/version-1.0.4/Overview.md",42365],c0391845:[()=>n.e(23234).then(n.bind(n,36422)),"@site/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",36422],c065fcb9:[()=>n.e(4800).then(n.bind(n,66225)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",66225],c14fdd92:[()=>n.e(19042).then(n.bind(n,72368)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",72368],c23b5e26:[()=>n.e(75658).then(n.bind(n,75410)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",75410],c2492d56:[()=>n.e(84202).then(n.bind(n,35967)),"@site/versioned_docs/version-0.11.4/Get Started/Create a Spark Cluster.md",35967],c2e40c34:[()=>n.e(85122).then(n.bind(n,99967)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",99967],c2e57a1b:[()=>n.e(26685).then(n.bind(n,39133)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Text Explainers.md",39133],c34b1e36:[()=>n.e(83251).then(n.bind(n,54824)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",54824],c38c658c:[()=>n.e(62565).then(n.bind(n,91960)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",91960],c39e3602:[()=>Promise.all([n.e(40532),n.e(46807)]).then(n.bind(n,34717)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_opencv.md",34717],c3b1d949:[()=>n.e(35056).then(n.bind(n,84577)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",84577],c3c516ff:[()=>n.e(65752).then(n.bind(n,68928)),"@site/blog/overview.md?truncated=true",68928],c3dac62d:[()=>n.e(35537).then(n.bind(n,8045)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",8045],c40984d1:[()=>Promise.all([n.e(40532),n.e(87320)]).then(n.bind(n,64356)),"@site/versioned_docs/version-0.11.3/Quick Examples/estimators/estimators_causal.md",64356],c413b43a:[()=>n.e(89767).then(n.bind(n,91730)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",91730],c4d09a44:[()=>n.e(52406).then(n.bind(n,48327)),"@site/versioned_docs/version-0.11.3/Get Started/Quickstart - Your First Models.md",48327],c4d68582:[()=>n.e(429).then(n.bind(n,2351)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",2351],c4f5d8e4:[()=>Promise.all([n.e(40532),n.e(87780),n.e(64195)]).then(n.bind(n,35567)),"@site/src/pages/index.js",35567],c5775233:[()=>n.e(47454).then(n.bind(n,21794)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenAI/OpenAI.md",21794],c5c95e45:[()=>n.e(24422).then(n.bind(n,73290)),"@site/docs/Explore Algorithms/Responsible AI/Image Explainers.md",73290],c73cfc52:[()=>n.e(18825).then(n.bind(n,75084)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/OpenCV/Image Transformations.md",75084],c764f2c0:[()=>n.e(97030).then(n.bind(n,37506)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Langchain.md",37506],c76e8cac:[()=>n.e(11298).then(n.bind(n,22344)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",22344],c82134d8:[()=>n.e(29540).then(n.bind(n,50500)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",50500],c90c942b:[()=>Promise.all([n.e(40532),n.e(91824)]).then(n.bind(n,21475)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_core.md",21475],c95882d3:[()=>n.e(31153).then(n.bind(n,19210)),"@site/docs/Use with MLFlow/Autologging.md",19210],c991f47b:[()=>n.e(68549).then(n.bind(n,6034)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor.md",6034],ca0a1bad:[()=>n.e(55305).then(n.bind(n,22468)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",22468],cb5f1a06:[()=>n.e(54528).then(n.bind(n,15346)),"@site/blog/overview.md",15346],cb7c2a83:[()=>n.e(21933).then(n.bind(n,24630)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",24630],cbc3190d:[()=>n.e(75488).then(n.bind(n,85418)),"@site/docs/Overview.md",85418],cc52e3cd:[()=>n.e(91214).then(n.bind(n,70650)),"@site/versioned_docs/version-1.0.4/Get Started/Create a Spark Cluster.md",70650],ccc49370:[()=>Promise.all([n.e(40532),n.e(24572),n.e(46048),n.e(46103)]).then(n.bind(n,65203)),"@theme/BlogPostPage",65203],cd8e97a5:[()=>n.e(44422).then(n.t.bind(n,53485,19)),"~docs/default/version-1-0-1-metadata-prop-e87.json",53485],ce40236b:[()=>n.e(27497).then(n.bind(n,18347)),"@site/versioned_docs/version-1.0.4/Get Started/Set up Cognitive Services.md",18347],ce7af6ae:[()=>n.e(55864).then(n.bind(n,7156)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Responsible AI/Explanation Dashboard.md",7156],cee4c3b9:[()=>n.e(25844).then(n.bind(n,30877)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",30877],ceeda143:[()=>n.e(18443).then(n.bind(n,31146)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",31146],cf57716c:[()=>n.e(5e3).then(n.bind(n,74832)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",74832],cf666d49:[()=>Promise.all([n.e(40532),n.e(25449)]).then(n.bind(n,34621)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_causal.md",34621],cff2e56e:[()=>n.e(56373).then(n.bind(n,65611)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search.md",65611],d05981ec:[()=>n.e(12124).then(n.bind(n,94929)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Geospatial Services.md",94929],d0c5e4fa:[()=>Promise.all([n.e(40532),n.e(95717)]).then(n.bind(n,77718)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_deep_learning.md",77718],d0cb74df:[()=>n.e(94597).then(n.bind(n,82689)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms.md",82689],d1246f49:[()=>Promise.all([n.e(40532),n.e(69966)]).then(n.bind(n,16082)),"@site/versioned_docs/version-0.11.4/Quick Examples/estimators/estimators_lightgbm.md",16082],d13e532e:[()=>n.e(8146).then(n.bind(n,90398)),"@site/docs/Explore Algorithms/Responsible AI/Text Explainers.md",90398],d1428557:[()=>n.e(3765).then(n.bind(n,25792)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenAI/OpenAI.md",25792],d175df5e:[()=>n.e(92805).then(n.t.bind(n,7812,19)),"~blog/default/synapse-ml-blog-archive-c02.json",7812],d252041b:[()=>n.e(76300).then(n.bind(n,29685)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Distributed Training.md",29685],d25cde70:[()=>n.e(81102).then(n.bind(n,96397)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk.md",96397],d3093636:[()=>n.e(77491).then(n.bind(n,94648)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",94648],d355047f:[()=>n.e(64713).then(n.bind(n,77696)),"@site/versioned_docs/version-0.11.4/Use with MLFlow/Overview.md",77696],d36f3429:[()=>n.e(91387).then(n.bind(n,87735)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Overview.md",87735],d39aa6d3:[()=>n.e(57930).then(n.bind(n,91123)),"@site/versioned_docs/version-0.11.3/Use with MLFlow/Install.md",91123],d3c9b0d7:[()=>n.e(98050).then(n.bind(n,9151)),"@site/versioned_docs/version-0.11.3/Get Started/Install SynapseML.md",9151],d41234a3:[()=>n.e(44177).then(n.bind(n,2193)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format.md",2193],d535b358:[()=>n.e(88984).then(n.bind(n,62051)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",62051],d57515f7:[()=>n.e(37638).then(n.bind(n,18898)),"@site/versioned_docs/version-1.0.3/Reference/R Setup.md",18898],d620a661:[()=>n.e(28746).then(n.bind(n,35215)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",35215],d707d2ed:[()=>n.e(83669).then(n.bind(n,35018)),"@site/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",35018],d7f58694:[()=>n.e(94553).then(n.bind(n,32484)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding.md",32484],d81d1137:[()=>n.e(44490).then(n.bind(n,81287)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",81287],d84904f9:[()=>n.e(18563).then(n.bind(n,67607)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers.md",67607],d8b3ac6a:[()=>n.e(94330).then(n.bind(n,7627)),"@site/docs/Explore Algorithms/Other Algorithms/Cyber ML.md",7627],d8d6ba90:[()=>n.e(51842).then(n.t.bind(n,83769,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-plugin-content-docs/default/plugin-route-context-module-100.json",83769],d95a70ea:[()=>n.e(62721).then(n.bind(n,32398)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/OpenCV/Image Transformations.md",32398],d9837698:[()=>n.e(57168).then(n.bind(n,80927)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/LightGBM/Overview.md",80927],d989c707:[()=>n.e(58462).then(n.bind(n,66657)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Overview.md",66657],d9e5e528:[()=>Promise.all([n.e(40532),n.e(38057)]).then(n.bind(n,58319)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_cognitive.md",58319],da508cef:[()=>n.e(33064).then(n.bind(n,16685)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenCV/Image Transformations.md",16685],dac82c00:[()=>Promise.all([n.e(40532),n.e(28559)]).then(n.bind(n,21888)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_cognitive.md",21888],db170a23:[()=>n.e(67706).then(n.bind(n,15710)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers.md",15710],dc1e40d7:[()=>n.e(27991).then(n.t.bind(n,5244,19)),"~docs/default/version-1-0-2-metadata-prop-290.json",5244],dc7f7ac1:[()=>n.e(92018).then(n.bind(n,82058)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",82058],dd29718b:[()=>n.e(15579).then(n.bind(n,6692)),"@site/versioned_docs/version-0.11.4/Reference/Developer Setup.md",6692],dd3f9ada:[()=>n.e(17820).then(n.bind(n,66851)),"@site/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN.md",66851],ddacd194:[()=>n.e(62465).then(n.bind(n,49718)),"@site/versioned_docs/version-1.0.3/Deploy Models/Overview.md",49718],de3bb7c3:[()=>n.e(42970).then(n.bind(n,13725)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",13725],dea122e2:[()=>n.e(99781).then(n.bind(n,77541)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",77541],deef0af8:[()=>n.e(89407).then(n.bind(n,41983)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification.md",41983],df01388f:[()=>Promise.all([n.e(40532),n.e(36828)]).then(n.bind(n,36579)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_vw.md",36579],dfb6dfe9:[()=>n.e(59076).then(n.bind(n,48770)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits.md",48770],e0077969:[()=>n.e(97297).then(n.bind(n,43415)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Deep Learning/Distributed Training.md",43415],e03f5a54:[()=>n.e(82569).then(n.bind(n,44581)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Cyber ML.md",44581],e1269457:[()=>n.e(80598).then(n.bind(n,10036)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",10036],e12b4691:[()=>n.e(10912).then(n.bind(n,19252)),"@site/versioned_docs/version-1.0.2/Deploy Models/Overview.md",19252],e1b2b5f3:[()=>n.e(81340).then(n.bind(n,60269)),"@site/blog/2018-04-01-Flexible and Scalable Deep Learning with MMLSpark.md",60269],e1ed1e14:[()=>n.e(82880).then(n.bind(n,3721)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/OpenCV/Image Transformations.md",3721],e21a3367:[()=>n.e(42638).then(n.bind(n,8940)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Geospatial Services.md",8940],e24ee294:[()=>Promise.all([n.e(40532),n.e(8040)]).then(n.bind(n,73198)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_core.md",73198],e2689dfc:[()=>n.e(57258).then(n.bind(n,10290)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions.md",10290],e2ce57ed:[()=>n.e(91210).then(n.bind(n,50102)),"@site/blog/2019-06-01-MMLSpark Unifying Machine Learning Ecosystems at Massive Scales.md",50102],e3089280:[()=>n.e(42283).then(n.bind(n,88617)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/Langchain.md",88617],e316de39:[()=>n.e(22011).then(n.bind(n,74053)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier.md",74053],e402e26d:[()=>Promise.all([n.e(40532),n.e(94623)]).then(n.bind(n,2570)),"@site/versioned_docs/version-1.0.3/Quick Examples/transformers/transformers_cognitive.md",2570],e464d112:[()=>n.e(94508).then(n.bind(n,50791)),"@site/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection.md",50791],e4ba4487:[()=>n.e(81787).then(n.bind(n,43982)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",43982],e4e95844:[()=>n.e(2797).then(n.bind(n,16230)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",16230],e579bad6:[()=>n.e(24627).then(n.bind(n,18579)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression.md",18579],e5c4ef01:[()=>n.e(96673).then(n.bind(n,95705)),"@site/versioned_docs/version-0.11.4/Reference/Docker Setup.md",95705],e5fe80e3:[()=>n.e(66419).then(n.bind(n,60673)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection.md",60673],e7250a93:[()=>n.e(75586).then(n.bind(n,41630)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects.md",41630],e7bd83ca:[()=>Promise.all([n.e(40532),n.e(8168)]).then(n.bind(n,51453)),"@site/versioned_docs/version-1.0.1/Quick Examples/estimators/estimators_vw.md",51453],e84755e1:[()=>n.e(68283).then(n.bind(n,69821)),"@site/versioned_docs/version-1.0.4/Get Started/Quickstart - Your First Models.md",69821],e863531f:[()=>n.e(66981).then(n.bind(n,51460)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",51460],e887b273:[()=>n.e(34654).then(n.bind(n,65011)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects.md",65011],e8b9c2c7:[()=>n.e(84979).then(n.bind(n,28781)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",28781],e8d66fc1:[()=>n.e(271).then(n.bind(n,33942)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data.md",33942],e9aab890:[()=>n.e(51732).then(n.bind(n,48613)),"@site/versioned_docs/version-0.11.4/Reference/Contributor Guide.md",48613],e9b75d67:[()=>n.e(95064).then(n.bind(n,28314)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/OpenAI/OpenAI.md",28314],ea2d2bcc:[()=>n.e(4204).then(n.bind(n,63800)),"@site/docs/Explore Algorithms/Deep Learning/ONNX.md",63800],eb0ef952:[()=>n.e(98197).then(n.bind(n,89595)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM.md",89595],ebf648dd:[()=>Promise.all([n.e(40532),n.e(15566)]).then(n.bind(n,33154)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_vw.md",33154],ed316aed:[()=>n.e(67870).then(n.bind(n,71351)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt.md",71351],ed40d5f1:[()=>n.e(6712).then(n.bind(n,21782)),"@site/versioned_docs/version-1.0.3/Reference/Developer Setup.md",21782],ed5c6c48:[()=>n.e(52787).then(n.bind(n,99421)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",99421],ed6d544d:[()=>n.e(63448).then(n.bind(n,79176)),"@site/docs/Explore Algorithms/OpenAI/Langchain.md",79176],ee6cea3b:[()=>n.e(80829).then(n.bind(n,56433)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/AI Services/Geospatial Services.md",56433],ee83122d:[()=>n.e(85648).then(n.bind(n,25)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier.md",25],eed630e0:[()=>n.e(49101).then(n.bind(n,88069)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures.md",88069],ef6ec597:[()=>n.e(13482).then(n.bind(n,88889)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Deep Learning/Distributed Training.md",88889],ef77ef39:[()=>n.e(37392).then(n.bind(n,75405)),"@site/versioned_docs/version-1.0.2/Reference/Quickstart - LightGBM in Dotnet.md",75405],efbb59fb:[()=>n.e(99378).then(n.bind(n,47415)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",47415],efbc69e1:[()=>n.e(49196).then(n.bind(n,87949)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Causal Inference/Overview.md",87949],efc1978d:[()=>n.e(79079).then(n.bind(n,25012)),"@site/versioned_docs/version-1.0.2/Deploy Models/Quickstart - Deploying a Classifier.md",25012],f027bb15:[()=>n.e(36916).then(n.bind(n,53307)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Image Explainers.md",53307],f0649d54:[()=>n.e(60880).then(n.bind(n,82604)),"@site/versioned_docs/version-1.0.3/Reference/Quickstart - LightGBM in Dotnet.md",82604],f14633e7:[()=>Promise.all([n.e(40532),n.e(84207)]).then(n.bind(n,61966)),"@site/versioned_docs/version-1.0.2/Quick Examples/estimators/estimators_lightgbm.md",61966],f1b2dc7f:[()=>n.e(28634).then(n.bind(n,56309)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance.md",56309],f21c13c4:[()=>n.e(23807).then(n.bind(n,42334)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier.md",42334],f228fdc5:[()=>n.e(82643).then(n.bind(n,66874)),"@site/versioned_docs/version-1.0.3/Get Started/Create a Spark Cluster.md",66874],f264933a:[()=>n.e(1283).then(n.bind(n,99121)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection.md",99121],f322d2da:[()=>n.e(60445).then(n.bind(n,34609)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs.md",34609],f38a0cbd:[()=>Promise.all([n.e(40532),n.e(61829)]).then(n.bind(n,66349)),"@site/versioned_docs/version-1.0.2/Quick Examples/transformers/transformers_core.md",66349],f3e986ee:[()=>n.e(15788).then(n.bind(n,28310)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Deep Learning/Getting Started.md",28310],f537d1e9:[()=>n.e(54890).then(n.bind(n,5773)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",5773],f5b4e517:[()=>n.e(98546).then(n.bind(n,23715)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis.md",23715],f7580424:[()=>n.e(82158).then(n.bind(n,74640)),"@site/versioned_docs/version-1.0.1/Use with MLFlow/Autologging.md",74640],f7a32432:[()=>Promise.all([n.e(40532),n.e(84954)]).then(n.bind(n,19193)),"@site/docs/Quick Examples/transformers/transformers_deep_learning.md",19193],f7bfff73:[()=>Promise.all([n.e(40532),n.e(94603)]).then(n.bind(n,81363)),"@site/versioned_docs/version-0.11.4/Quick Examples/transformers/transformers_opencv.md",81363],f8597cb0:[()=>n.e(82744).then(n.bind(n,25907)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests.md",25907],f85db098:[()=>Promise.all([n.e(40532),n.e(95049)]).then(n.bind(n,46420)),"@site/docs/Reference/Dotnet Setup.md",46420],f8aab3ca:[()=>n.e(84880).then(n.bind(n,29469)),"@site/versioned_docs/version-1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",29469],fa2e2688:[()=>n.e(64927).then(n.bind(n,78030)),"@site/docs/Explore Algorithms/AI Services/Overview.md",78030],fa889bee:[()=>n.e(81878).then(n.bind(n,91966)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression.md",91966],faca745a:[()=>Promise.all([n.e(40532),n.e(67664)]).then(n.bind(n,44565)),"@site/versioned_docs/version-1.0.3/Quick Examples/estimators/estimators_core.md",44565],fba72e87:[()=>n.e(11542).then(n.bind(n,20970)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification.md",20970],fbb011bb:[()=>n.e(83378).then(n.bind(n,20693)),"@site/docs/Use with MLFlow/Install.md",20693],fc338b02:[()=>n.e(5378).then(n.t.bind(n,7085,19)),"/home/vsts/work/1/s/website/.docusaurus/docusaurus-theme-search-algolia/default/plugin-route-context-module-100.json",7085],fc47fc53:[()=>n.e(99601).then(n.bind(n,44088)),"@site/versioned_docs/version-1.0.1/Get Started/Install SynapseML.md",44088],fc598b5d:[()=>n.e(50002).then(n.bind(n,89076)),"@site/versioned_docs/version-1.0.4/Explore Algorithms/Vowpal Wabbit/Overview.md",89076],fd3b6ea5:[()=>n.e(31279).then(n.bind(n,7922)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes.md",7922],fd4bd09d:[()=>n.e(1286).then(n.bind(n,70781)),"@site/versioned_docs/version-0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier.md",70781],fd748117:[()=>n.e(68809).then(n.bind(n,1359)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text.md",1359],fe2c893b:[()=>n.e(50297).then(n.bind(n,18859)),"@site/versioned_docs/version-0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier.md",18859],fe5d19cb:[()=>n.e(97065).then(n.bind(n,4460)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key.md",4460],fe6131e8:[()=>n.e(44539).then(n.bind(n,17054)),"@site/versioned_docs/version-1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations.md",17054],fece3c24:[()=>Promise.all([n.e(40532),n.e(40367)]).then(n.bind(n,88387)),"@site/versioned_docs/version-1.0.1/Quick Examples/transformers/transformers_core.md",88387],ff428354:[()=>n.e(51078).then(n.bind(n,92498)),"@site/versioned_docs/version-1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors.md",92498],ffb7f88a:[()=>n.e(85433).then(n.t.bind(n,81421,19)),"~docs/default/version-0-11-3-metadata-prop-9e0.json",81421]};function l(e){let{error:t,retry:n,pastDelay:r}=e;return t?s.createElement("div",{style:{textAlign:"center",color:"#fff",backgroundColor:"#fa383e",borderColor:"#fa383e",borderStyle:"solid",borderRadius:"0.25rem",borderWidth:"1px",boxSizing:"border-box",display:"block",padding:"1rem",flex:"0 0 50%",marginLeft:"25%",marginRight:"25%",marginTop:"5rem",maxWidth:"50%",width:"100%"}},s.createElement("p",null,String(t)),s.createElement("div",null,s.createElement("button",{type:"button",onClick:n},"Retry"))):r?s.createElement("div",{style:{display:"flex",justifyContent:"center",alignItems:"center",height:"100vh"}},s.createElement("svg",{id:"loader",style:{width:128,height:110,position:"absolute",top:"calc(100vh - 64%)"},viewBox:"0 0 45 45",xmlns:"http://www.w3.org/2000/svg",stroke:"#61dafb"},s.createElement("g",{fill:"none",fillRule:"evenodd",transform:"translate(1 1)",strokeWidth:"2"},s.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},s.createElement("animate",{attributeName:"r",begin:"1.5s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-opacity",begin:"1.5s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-width",begin:"1.5s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),s.createElement("circle",{cx:"22",cy:"22",r:"6",strokeOpacity:"0"},s.createElement("animate",{attributeName:"r",begin:"3s",dur:"3s",values:"6;22",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-opacity",begin:"3s",dur:"3s",values:"1;0",calcMode:"linear",repeatCount:"indefinite"}),s.createElement("animate",{attributeName:"stroke-width",begin:"3s",dur:"3s",values:"2;0",calcMode:"linear",repeatCount:"indefinite"})),s.createElement("circle",{cx:"22",cy:"22",r:"8"},s.createElement("animate",{attributeName:"r",begin:"0s",dur:"1.5s",values:"6;1;2;3;4;5;6",calcMode:"linear",repeatCount:"indefinite"}))))):null}var d=n(99670),p=n(30226);function u(e,t){if("*"===e)return a()({loading:l,loader:()=>Promise.all([n.e(40532),n.e(53140)]).then(n.bind(n,53140)),modules:["@theme/NotFound"],webpack:()=>[53140],render(e,t){const n=e.default;return s.createElement(p.z,{value:{plugin:{name:"native",id:"default"}}},s.createElement(n,t))}});const o=i[e+"-"+t],u={},m=[],h=[],g=(0,d.Z)(o);return Object.entries(g).forEach((e=>{let[t,n]=e;const s=c[n];s&&(u[t]=s[0],m.push(s[1]),h.push(s[2]))})),a().Map({loading:l,loader:u,modules:m,webpack:()=>h,render(t,n){const a=JSON.parse(JSON.stringify(o));Object.entries(t).forEach((t=>{let[n,s]=t;const r=s.default;if(!r)throw new Error("The page component at "+e+" doesn't have a default export. This makes it impossible to render anything. Consider default-exporting a React component.");"object"!=typeof r&&"function"!=typeof r||Object.keys(s).filter((e=>"default"!==e)).forEach((e=>{r[e]=s[e]}));let o=a;const i=n.split(".");i.slice(0,-1).forEach((e=>{o=o[e]})),o[i[i.length-1]]=r}));const i=a.__comp;delete a.__comp;const c=a.__context;return delete a.__context,s.createElement(p.z,{value:c},s.createElement(i,(0,r.Z)({},a,n)))}})}const m=[{path:"/SynapseML/blog/",component:u("/SynapseML/blog/","32c"),exact:!0},{path:"/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/",component:u("/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/","ecd"),exact:!0},{path:"/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/",component:u("/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/","870"),exact:!0},{path:"/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/",component:u("/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/","83d"),exact:!0},{path:"/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/",component:u("/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/","347"),exact:!0},{path:"/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/",component:u("/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/","546"),exact:!0},{path:"/SynapseML/blog/archive/",component:u("/SynapseML/blog/archive/","140"),exact:!0},{path:"/SynapseML/blog/overview/",component:u("/SynapseML/blog/overview/","32f"),exact:!0},{path:"/SynapseML/search/",component:u("/SynapseML/search/","a70"),exact:!0},{path:"/SynapseML/videos/",component:u("/SynapseML/videos/","9fc"),exact:!0},{path:"/SynapseML/docs/0.11.3/",component:u("/SynapseML/docs/0.11.3/","649"),routes:[{path:"/SynapseML/docs/0.11.3/Deploy Models/Overview/",component:u("/SynapseML/docs/0.11.3/Deploy Models/Overview/","107"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/","7ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","428"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/","20d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","dd0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/","8b1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","7f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/","9df"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/","398"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","475"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","746"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","00a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","be4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","4ed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/","f94"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","af8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","c3d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","1f2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/","d3c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","b90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","583"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/","c90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/","59d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/","4d0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","b98"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","52d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","018"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","086"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","2e6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","cd5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/","7d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","c77"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/","f47"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/","ffd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","c37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","3dc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/","5b7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/","e89"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","22c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","6da"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","4f5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/","b53"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/","e82"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","eb4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/","aa0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/","547"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/","ca9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","1c8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","850"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","103"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","22a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/","4a3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/","51c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","79d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","dbd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/","f6c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","899"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","de1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","6e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/","604"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Install SynapseML/",component:u("/SynapseML/docs/0.11.3/Get Started/Install SynapseML/","53d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/","bcf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/","e8a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Overview/",component:u("/SynapseML/docs/0.11.3/Overview/","201"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/","f33"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/","d03"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/","0fc"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/","113"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/","bcf"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/","3f4"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/","abc"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/","b7d"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/","a66"),exact:!0},{path:"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/","b96"),exact:!0},{path:"/SynapseML/docs/0.11.3/Reference/Contributor Guide/",component:u("/SynapseML/docs/0.11.3/Reference/Contributor Guide/","06f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Developer Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Developer Setup/","cfc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Docker Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Docker Setup/","419"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Dotnet Setup/",component:u("/SynapseML/docs/0.11.3/Reference/Dotnet Setup/","46d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/","9b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Reference/R Setup/",component:u("/SynapseML/docs/0.11.3/Reference/R Setup/","1a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/","6a1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Install/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Install/","aa4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.3/Use with MLFlow/Overview/",component:u("/SynapseML/docs/0.11.3/Use with MLFlow/Overview/","a43"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/0.11.4/",component:u("/SynapseML/docs/0.11.4/","72b"),routes:[{path:"/SynapseML/docs/0.11.4/Deploy Models/Overview/",component:u("/SynapseML/docs/0.11.4/Deploy Models/Overview/","1a4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/","ceb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","f98"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/","9e7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","2c3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/","be8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","1d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/","31e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/","8c4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","4f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","d9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","e8f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","8e2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/","78b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","756"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","264"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","c8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/","247"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","c00"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","c42"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/","53f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/","2d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/","169"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","b68"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","ff8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","016"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","6a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","b11"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/","e22"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","934"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/","fea"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/","cfe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","78f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","163"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/","29e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/","747"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","faf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","566"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","491"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/","7a4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/","75f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","2b0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/","f95"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/","66b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/","e63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","783"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","e01"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","3b8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","80b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/","18e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/","128"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","02f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","6d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/","b97"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","6ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","c86"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","964"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/","cb4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Install SynapseML/",component:u("/SynapseML/docs/0.11.4/Get Started/Install SynapseML/","34f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/","6fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/","528"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Overview/",component:u("/SynapseML/docs/0.11.4/Overview/","83a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/","32e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/","269"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/","11e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/","c94"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/","bf0"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/","728"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/","12e"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/","1cc"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/","8fb"),exact:!0},{path:"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/","828"),exact:!0},{path:"/SynapseML/docs/0.11.4/Reference/Contributor Guide/",component:u("/SynapseML/docs/0.11.4/Reference/Contributor Guide/","881"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Developer Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Developer Setup/","171"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Docker Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Docker Setup/","f85"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Dotnet Setup/",component:u("/SynapseML/docs/0.11.4/Reference/Dotnet Setup/","f28"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/","553"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Reference/R Setup/",component:u("/SynapseML/docs/0.11.4/Reference/R Setup/","fd3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/","3cb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Install/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Install/","360"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/0.11.4/Use with MLFlow/Overview/",component:u("/SynapseML/docs/0.11.4/Use with MLFlow/Overview/","4e1"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.1/",component:u("/SynapseML/docs/1.0.1/","c9e"),routes:[{path:"/SynapseML/docs/1.0.1/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.1/Deploy Models/Overview/","7aa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/","c4a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","d96"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/","a96"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","4e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/","a84"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","e94"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/","be5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","76f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","5fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","417"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","cb0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","831"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","ce6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/","a37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","0db"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","d46"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","b74"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/","109"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","de7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","250"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/","68a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/","920"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/","4a5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","a53"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9d3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","da8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","b25"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","18d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","f90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/","15c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","796"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/","045"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/","5e3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","6f5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","457"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/","f73"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/","0f7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","fa5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","201"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","bd7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/","979"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/","810"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","1e9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/","6f0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/","55c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/","e7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","df0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","94a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","41d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","d72"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/","895"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/","b41"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","c46"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","728"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/","21f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","a43"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","7db"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","9d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/","697"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.1/Get Started/Install SynapseML/","349"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/","d39"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/","2fa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Overview/",component:u("/SynapseML/docs/1.0.1/Overview/","64e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/","8d4"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/","d28"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/","1b9"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/","b23"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/","86c"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/","03e"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/","0db"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/","f4e"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/","945"),exact:!0},{path:"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/","369"),exact:!0},{path:"/SynapseML/docs/1.0.1/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.1/Reference/Contributor Guide/","5ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Developer Setup/","971"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Docker Setup/","e6e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.1/Reference/Dotnet Setup/","179"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/","049"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Reference/R Setup/",component:u("/SynapseML/docs/1.0.1/Reference/R Setup/","392"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/","23e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Install/","0df"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.1/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.1/Use with MLFlow/Overview/","d32"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.2/",component:u("/SynapseML/docs/1.0.2/","df4"),routes:[{path:"/SynapseML/docs/1.0.2/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.2/Deploy Models/Overview/","a7c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/","7f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","185"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/","fa7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","1b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/","78e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","400"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/","5da"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","286"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","c7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","1d5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","d5d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","46e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","4ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/","8e5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","405"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","2d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","764"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/","f3b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","8c6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","40b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/","a43"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/","001"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/","a2e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","613"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","5c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","893"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","bb7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","ccf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","a9c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/","108"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","a01"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/","f2d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/","a4a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","a0a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","5bb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/","6ae"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/","4c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","056"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","a63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","1e8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/","6cd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/","318"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","a37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/","eed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/","25a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/","810"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","5c9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","b99"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","3dd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","1bd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/","0b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/","250"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","bd7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","b2f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/","5a7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","4ab"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","d16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","157"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/","3f8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.2/Get Started/Install SynapseML/","01e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/","0aa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/","095"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Overview/",component:u("/SynapseML/docs/1.0.2/Overview/","c78"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/","348"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/","386"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/","585"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/","fa7"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/","b73"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/","762"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/","7ec"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/","bad"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/","183"),exact:!0},{path:"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/","029"),exact:!0},{path:"/SynapseML/docs/1.0.2/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.2/Reference/Contributor Guide/","8f1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Developer Setup/","5bf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Docker Setup/","294"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.2/Reference/Dotnet Setup/","b72"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/","595"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Reference/R Setup/",component:u("/SynapseML/docs/1.0.2/Reference/R Setup/","a22"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/","e9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Install/","7d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.2/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.2/Use with MLFlow/Overview/","e3c"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/1.0.3/",component:u("/SynapseML/docs/1.0.3/","60e"),routes:[{path:"/SynapseML/docs/1.0.3/Deploy Models/Overview/",component:u("/SynapseML/docs/1.0.3/Deploy Models/Overview/","1cf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/","695"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","0c3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/","e88"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","6d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/","0d9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","5bc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/","3ee"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","7b8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","5f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","85d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","6e8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","e1a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","54d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/","562"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","734"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","410"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","823"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/","1ea"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","96b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","432"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/","f57"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/","141"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/","005"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","2d6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","abe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","fb6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","1fe"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","561"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","158"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/","e1b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","99d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/","1f6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/","b8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","3b4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","146"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/","57d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/","e16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","3d8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","30f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","e7e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/","8c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/","1f6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","ea9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/","211"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/","18d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/","d3d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","025"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","878"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","326"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","a63"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/","8ed"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/","893"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","747"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","484"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/","60f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","c16"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","35d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","f74"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/","ed3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Install SynapseML/",component:u("/SynapseML/docs/1.0.3/Get Started/Install SynapseML/","df3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/","450"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/","dff"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Overview/",component:u("/SynapseML/docs/1.0.3/Overview/","094"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/","505"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/","6ff"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/","104"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/","08b"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/","74d"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/","baf"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/","dcc"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/","80b"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/","4af"),exact:!0},{path:"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/","9cd"),exact:!0},{path:"/SynapseML/docs/1.0.3/Reference/Contributor Guide/",component:u("/SynapseML/docs/1.0.3/Reference/Contributor Guide/","676"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Developer Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Developer Setup/","904"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Docker Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Docker Setup/","963"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Dotnet Setup/",component:u("/SynapseML/docs/1.0.3/Reference/Dotnet Setup/","c7f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/","9f9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Reference/R Setup/",component:u("/SynapseML/docs/1.0.3/Reference/R Setup/","3a0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/","00b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Install/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Install/","5a1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/1.0.3/Use with MLFlow/Overview/",component:u("/SynapseML/docs/1.0.3/Use with MLFlow/Overview/","234"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/next/",component:u("/SynapseML/docs/next/","3d8"),routes:[{path:"/SynapseML/docs/next/Deploy Models/Overview/",component:u("/SynapseML/docs/next/Deploy Models/Overview/","29d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/","eeb"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","12a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/","00a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","e9b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/","0d4"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","6d0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/","bef"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","4f8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","3a8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","385"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5c7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","f4f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","bf9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/","89c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","fc9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","80d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/",component:u("/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/","b09"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","12f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/","86b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","25e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","c31"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/","1a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/","270"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/","c9a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","fdc"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","9cf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","a7b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","641"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","6e0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","8d1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/","50f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","d9a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/","ef5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/","43a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/","a4b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","a86"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","101"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/","5d7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/","1e7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","88e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","d5d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","41e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/","d73"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/","267"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","95b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/","798"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/","0ec"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/","039"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","2fd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","8f3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","484"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","3d2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/","caf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/","219"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","b35"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","c0f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/","fae"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","3c9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","71e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","e9d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/next/Get Started/Create a Spark Cluster/","37f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Install SynapseML/",component:u("/SynapseML/docs/next/Get Started/Install SynapseML/","975"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/next/Get Started/Quickstart - Your First Models/","8a6"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/next/Get Started/Set up Cognitive Services/","47d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Overview/",component:u("/SynapseML/docs/next/Overview/","9a5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/","c47"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/","6e1"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_core/","7e6"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/","5eb"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/","49f"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/","443"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_core/","0a9"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/","137"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/","3f4"),exact:!0},{path:"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/","d93"),exact:!0},{path:"/SynapseML/docs/next/Reference/Contributor Guide/",component:u("/SynapseML/docs/next/Reference/Contributor Guide/","dec"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Developer Setup/",component:u("/SynapseML/docs/next/Reference/Developer Setup/","1c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Docker Setup/",component:u("/SynapseML/docs/next/Reference/Docker Setup/","598"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Reference/Dotnet Setup/",component:u("/SynapseML/docs/next/Reference/Dotnet Setup/","ded"),exact:!0},{path:"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/","8a8"),exact:!0},{path:"/SynapseML/docs/next/Reference/R Setup/",component:u("/SynapseML/docs/next/Reference/R Setup/","d8b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/next/Use with MLFlow/Autologging/","0ca"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Install/",component:u("/SynapseML/docs/next/Use with MLFlow/Install/","9b7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/next/Use with MLFlow/Overview/",component:u("/SynapseML/docs/next/Use with MLFlow/Overview/","b10"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/docs/",component:u("/SynapseML/docs/","352"),routes:[{path:"/SynapseML/docs/Deploy Models/Overview/",component:u("/SynapseML/docs/Deploy Models/Overview/","438"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/",component:u("/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/","e9f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/","3f1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/","c3c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/","de7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Overview/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Overview/","52c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/","dc7"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/","2c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/","f90"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/","425"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/","d37"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/","5e1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/",component:u("/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/","eb1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",component:u("/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/","7e3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/","75a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/","b1b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/","cf3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/",component:u("/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/","bcf"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/","3e2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/","618"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/","7c2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/",component:u("/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/","a7a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/","1ba"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/","628"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/","90b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/","57b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/","b77"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/","a81"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/",component:u("/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/","93c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",component:u("/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/","87a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",component:u("/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/","699"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/LightGBM/Overview/",component:u("/SynapseML/docs/Explore Algorithms/LightGBM/Overview/","300"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/",component:u("/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/","5c1"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/","06d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/","3b0"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/","b8b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/","d6d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",component:u("/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/","baa"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/",component:u("/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/","179"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/","4b2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/","54a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/","572"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/",component:u("/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/","743"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/","82e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/","dc2"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/",component:u("/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/","72f"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/","e8d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/","e6c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/","f44"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/","9d9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/","082"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/","f62"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/","804"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/","60b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/",component:u("/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/","3c5"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/","32b"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/","046"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/","5b3"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/","490"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/","5dd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/",component:u("/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/","804"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Create a Spark Cluster/",component:u("/SynapseML/docs/Get Started/Create a Spark Cluster/","37d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Install SynapseML/",component:u("/SynapseML/docs/Get Started/Install SynapseML/","16c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Quickstart - Your First Models/",component:u("/SynapseML/docs/Get Started/Quickstart - Your First Models/","e0c"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Get Started/Set up Cognitive Services/",component:u("/SynapseML/docs/Get Started/Set up Cognitive Services/","ea8"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Overview/",component:u("/SynapseML/docs/Overview/","58e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_causal/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_causal/","e9d"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/","419"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_core/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_core/","63c"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/","d43"),exact:!0},{path:"/SynapseML/docs/Quick Examples/estimators/estimators_vw/",component:u("/SynapseML/docs/Quick Examples/estimators/estimators_vw/","bd9"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/","bbd"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_core/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_core/","1a1"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/","483"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_opencv/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_opencv/","8a0"),exact:!0},{path:"/SynapseML/docs/Quick Examples/transformers/transformers_vw/",component:u("/SynapseML/docs/Quick Examples/transformers/transformers_vw/","fd7"),exact:!0},{path:"/SynapseML/docs/Reference/Contributor Guide/",component:u("/SynapseML/docs/Reference/Contributor Guide/","6e9"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Developer Setup/",component:u("/SynapseML/docs/Reference/Developer Setup/","367"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Docker Setup/",component:u("/SynapseML/docs/Reference/Docker Setup/","a2a"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Dotnet Setup/",component:u("/SynapseML/docs/Reference/Dotnet Setup/","995"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/",component:u("/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/","e58"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Reference/R Setup/",component:u("/SynapseML/docs/Reference/R Setup/","d4d"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Autologging/",component:u("/SynapseML/docs/Use with MLFlow/Autologging/","1cd"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Install/",component:u("/SynapseML/docs/Use with MLFlow/Install/","a7e"),exact:!0,sidebar:"docs"},{path:"/SynapseML/docs/Use with MLFlow/Overview/",component:u("/SynapseML/docs/Use with MLFlow/Overview/","424"),exact:!0,sidebar:"docs"}]},{path:"/SynapseML/",component:u("/SynapseML/","6e1"),exact:!0},{path:"*",component:u("*")}]},98934:(e,t,n)=>{"use strict";n.d(t,{_:()=>r,t:()=>o});var s=n(67294);const r=s.createContext(!1);function o(e){let{children:t}=e;const[n,o]=(0,s.useState)(!1);return(0,s.useEffect)((()=>{o(!0)}),[]),s.createElement(r.Provider,{value:n},t)}},49383:(e,t,n)=>{"use strict";var s=n(67294),r=n(73935),o=n(73727),a=n(70405),i=n(10412);const c=[n(56657),n(32497),n(3310),n(18320),n(52295)];var l=n(723),d=n(16550),p=n(18790);function u(e){let{children:t}=e;return s.createElement(s.Fragment,null,t)}var m=n(83117),h=n(35742),g=n(52263),f=n(44996),b=n(86668),y=n(10833),x=n(94711),v=n(19727),A=n(43320),S=n(90197);function E(){const{i18n:{defaultLocale:e,localeConfigs:t}}=(0,g.Z)(),n=(0,x.l)();return s.createElement(h.Z,null,Object.entries(t).map((e=>{let[t,{htmlLang:r}]=e;return s.createElement("link",{key:t,rel:"alternate",href:n.createUrl({locale:t,fullyQualified:!0}),hrefLang:r})})),s.createElement("link",{rel:"alternate",href:n.createUrl({locale:e,fullyQualified:!0}),hrefLang:"x-default"}))}function L(e){let{permalink:t}=e;const{siteConfig:{url:n}}=(0,g.Z)(),r=function(){const{siteConfig:{url:e}}=(0,g.Z)(),{pathname:t}=(0,d.TH)();return e+(0,f.Z)(t)}(),o=t?""+n+t:r;return s.createElement(h.Z,null,s.createElement("meta",{property:"og:url",content:o}),s.createElement("link",{rel:"canonical",href:o}))}function M(){const{i18n:{currentLocale:e}}=(0,g.Z)(),{metadata:t,image:n}=(0,b.L)();return s.createElement(s.Fragment,null,s.createElement(h.Z,null,s.createElement("meta",{name:"twitter:card",content:"summary_large_image"}),s.createElement("body",{className:v.h})),n&&s.createElement(y.d,{image:n}),s.createElement(L,null),s.createElement(E,null),s.createElement(S.Z,{tag:A.HX,locale:e}),s.createElement(h.Z,null,t.map(((e,t)=>s.createElement("meta",(0,m.Z)({key:t},e))))))}const k=new Map;function _(e){if(k.has(e.pathname))return{...e,pathname:k.get(e.pathname)};if((0,p.f)(l.Z,e.pathname).some((e=>{let{route:t}=e;return!0===t.exact})))return k.set(e.pathname,e.pathname),e;const t=e.pathname.trim().replace(/(?:\/index)?\.html$/,"")||"/";return k.set(e.pathname,t),{...e,pathname:t}}var w=n(98934),I=n(58940);function C(e){for(var t=arguments.length,n=new Array(t>1?t-1:0),s=1;s<t;s++)n[s-1]=arguments[s];const r=c.map((t=>{var s,r;const o=null!=(s=null==(r=t.default)?void 0:r[e])?s:t[e];return null==o?void 0:o(...n)}));return()=>r.forEach((e=>null==e?void 0:e()))}const Q=function(e){let{children:t,location:n,previousLocation:r}=e;return(0,s.useLayoutEffect)((()=>{r!==n&&(!function(e){let{location:t,previousLocation:n}=e;if(!n)return;const s=t.pathname===n.pathname,r=t.hash===n.hash,o=t.search===n.search;if(s&&r&&!o)return;const{hash:a}=t;if(a){const e=decodeURIComponent(a.substring(1)),t=document.getElementById(e);null==t||t.scrollIntoView()}else window.scrollTo(0,0)}({location:n,previousLocation:r}),C("onRouteDidUpdate",{previousLocation:r,location:n}))}),[r,n]),t};function R(e){const t=Array.from(new Set([e,decodeURI(e)])).map((e=>(0,p.f)(l.Z,e))).flat();return Promise.all(t.map((e=>null==e.route.component.preload?void 0:e.route.component.preload())))}class D extends s.Component{constructor(e){super(e),this.previousLocation=void 0,this.routeUpdateCleanupCb=void 0,this.previousLocation=null,this.routeUpdateCleanupCb=i.Z.canUseDOM?C("onRouteUpdate",{previousLocation:null,location:this.props.location}):()=>{},this.state={nextRouteHasLoaded:!0}}shouldComponentUpdate(e,t){if(e.location===this.props.location)return t.nextRouteHasLoaded;const n=e.location;return this.previousLocation=this.props.location,this.setState({nextRouteHasLoaded:!1}),this.routeUpdateCleanupCb=C("onRouteUpdate",{previousLocation:this.previousLocation,location:n}),R(n.pathname).then((()=>{this.routeUpdateCleanupCb(),this.setState({nextRouteHasLoaded:!0})})).catch((e=>{console.warn(e),window.location.reload()})),!1}render(){const{children:e,location:t}=this.props;return s.createElement(Q,{previousLocation:this.previousLocation,location:t},s.createElement(d.AW,{location:t,render:()=>e}))}}const T=D,O="__docusaurus-base-url-issue-banner-container",P="__docusaurus-base-url-issue-banner-suggestion-container",F="__DOCUSAURUS_INSERT_BASEURL_BANNER";function N(e){return"\nwindow['"+F+"'] = true;\n\ndocument.addEventListener('DOMContentLoaded', maybeInsertBanner);\n\nfunction maybeInsertBanner() {\n  var shouldInsert = window['"+F+"'];\n  shouldInsert && insertBanner();\n}\n\nfunction insertBanner() {\n  var bannerContainer = document.getElementById('"+O+"');\n  if (!bannerContainer) {\n    return;\n  }\n  var bannerHtml = "+JSON.stringify(function(e){return'\n<div id="__docusaurus-base-url-issue-banner" style="border: thick solid red; background-color: rgb(255, 230, 179); margin: 20px; padding: 20px; font-size: 20px;">\n   <p style="font-weight: bold; font-size: 30px;">Your Docusaurus site did not load properly.</p>\n   <p>A very common reason is a wrong site <a href="https://docusaurus.io/docs/docusaurus.config.js/#baseUrl" style="font-weight: bold;">baseUrl configuration</a>.</p>\n   <p>Current configured baseUrl = <span style="font-weight: bold; color: red;">'+e+"</span> "+("/"===e?" (default value)":"")+'</p>\n   <p>We suggest trying baseUrl = <span id="'+P+'" style="font-weight: bold; color: green;"></span></p>\n</div>\n'}(e)).replace(/</g,"\\<")+";\n  bannerContainer.innerHTML = bannerHtml;\n  var suggestionContainer = document.getElementById('"+P+"');\n  var actualHomePagePath = window.location.pathname;\n  var suggestedBaseUrl = actualHomePagePath.substr(-1) === '/'\n        ? actualHomePagePath\n        : actualHomePagePath + '/';\n  suggestionContainer.innerHTML = suggestedBaseUrl;\n}\n"}function B(){const{siteConfig:{baseUrl:e}}=(0,g.Z)();return(0,s.useLayoutEffect)((()=>{window[F]=!1}),[]),s.createElement(s.Fragment,null,!i.Z.canUseDOM&&s.createElement(h.Z,null,s.createElement("script",null,N(e))),s.createElement("div",{id:O}))}function V(){const{siteConfig:{baseUrl:e,baseUrlIssueBanner:t}}=(0,g.Z)(),{pathname:n}=(0,d.TH)();return t&&n===e?s.createElement(B,null):null}function G(){const{siteConfig:{favicon:e,title:t,noIndex:n},i18n:{currentLocale:r,localeConfigs:o}}=(0,g.Z)(),a=(0,f.Z)(e),{htmlLang:i,direction:c}=o[r];return s.createElement(h.Z,null,s.createElement("html",{lang:i,dir:c}),s.createElement("title",null,t),s.createElement("meta",{property:"og:title",content:t}),s.createElement("meta",{name:"viewport",content:"width=device-width, initial-scale=1.0"}),n&&s.createElement("meta",{name:"robots",content:"noindex, nofollow"}),e&&s.createElement("link",{rel:"icon",href:a}))}var W=n(44763);function U(){const e=(0,p.H)(l.Z),t=(0,d.TH)();return s.createElement(W.Z,null,s.createElement(I.M,null,s.createElement(w.t,null,s.createElement(u,null,s.createElement(G,null),s.createElement(M,null),s.createElement(V,null),s.createElement(T,{location:_(t)},e)))))}var z=n(16887);const H=function(e){try{return document.createElement("link").relList.supports(e)}catch{return!1}}("prefetch")?function(e){return new Promise(((t,n)=>{var s,r;if("undefined"==typeof document)return void n();const o=document.createElement("link");o.setAttribute("rel","prefetch"),o.setAttribute("href",e),o.onload=()=>t(),o.onerror=()=>n();const a=null!=(s=document.getElementsByTagName("head")[0])?s:null==(r=document.getElementsByName("script")[0])?void 0:r.parentNode;null==a||a.appendChild(o)}))}:function(e){return new Promise(((t,n)=>{const s=new XMLHttpRequest;s.open("GET",e,!0),s.withCredentials=!0,s.onload=()=>{200===s.status?t():n()},s.send(null)}))};var j=n(99670);const $=new Set,Z=new Set,X=()=>{var e,t;return(null==(e=navigator.connection)?void 0:e.effectiveType.includes("2g"))||(null==(t=navigator.connection)?void 0:t.saveData)},K={prefetch(e){if(!(e=>!X()&&!Z.has(e)&&!$.has(e))(e))return!1;$.add(e);const t=(0,p.f)(l.Z,e).flatMap((e=>{return t=e.route.path,Object.entries(z).filter((e=>{let[n]=e;return n.replace(/-[^-]+$/,"")===t})).flatMap((e=>{let[,t]=e;return Object.values((0,j.Z)(t))}));var t}));return Promise.all(t.map((e=>{const t=n.gca(e);return t&&!t.includes("undefined")?H(t).catch((()=>{})):Promise.resolve()})))},preload:e=>!!(e=>!X()&&!Z.has(e))(e)&&(Z.add(e),R(e))},Y=Object.freeze(K);if(i.Z.canUseDOM){window.docusaurus=Y;const e=r.hydrate;R(window.location.pathname).then((()=>{e(s.createElement(a.B6,null,s.createElement(o.VK,null,s.createElement(U,null))),document.getElementById("__docusaurus"))}))}},58940:(e,t,n)=>{"use strict";n.d(t,{_:()=>d,M:()=>p});var s=n(67294),r=n(36809);const o=JSON.parse('{"docusaurus-plugin-google-gtag":{"default":{"trackingID":["G-RWPE0183E8"],"anonymizeIP":true,"id":"default"}},"docusaurus-plugin-content-docs":{"default":{"path":"/SynapseML/docs","versions":[{"name":"current","label":"Next","isLast":false,"path":"/SynapseML/docs/next","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/next/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","path":"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/next/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/next/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/next/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/next/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/next/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/next/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/next/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/next/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/next/Reference/Dotnet Setup"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet"},{"id":"Reference/R Setup","path":"/SynapseML/docs/next/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/next/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/next/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/next/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/next/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.4","label":"1.0.4","isLast":true,"path":"/SynapseML/docs","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","path":"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.3","label":"1.0.3","isLast":false,"path":"/SynapseML/docs/1.0.3","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.3/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.3/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.3/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.3/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.3/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.3/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.3/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.3/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.3/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.3/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.2","label":"1.0.2","isLast":false,"path":"/SynapseML/docs/1.0.2","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.2/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.2/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.2/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.2/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.2/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.2/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.2/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.2/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.2/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.2/Overview","label":"What is SynapseML?"}}}},{"name":"1.0.1","label":"1.0.1","isLast":false,"path":"/SynapseML/docs/1.0.1","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/1.0.1/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/1.0.1/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/1.0.1/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/1.0.1/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/1.0.1/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/1.0.1/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/1.0.1/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/1.0.1/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/1.0.1/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/1.0.1/Overview","label":"What is SynapseML?"}}}},{"name":"0.11.4","label":"0.11.4","isLast":false,"path":"/SynapseML/docs/0.11.4","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/0.11.4/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/0.11.4/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/0.11.4/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/0.11.4/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/0.11.4/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/0.11.4/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/0.11.4/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/0.11.4/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/0.11.4/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/0.11.4/Overview","label":"What is SynapseML?"}}}},{"name":"0.11.3","label":"0.11.3","isLast":false,"path":"/SynapseML/docs/0.11.3","mainDocId":"Overview","docs":[{"id":"Deploy Models/Overview","path":"/SynapseML/docs/0.11.3/Deploy Models/Overview","sidebar":"docs"},{"id":"Deploy Models/Quickstart - Deploying a Classifier","path":"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Geospatial Services","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Multivariate Anomaly Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Analyze Text","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Create Audiobooks","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Flooding Risk","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk","sidebar":"docs"},{"id":"Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","path":"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance","sidebar":"docs"},{"id":"Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects","sidebar":"docs"},{"id":"Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Train Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data","sidebar":"docs"},{"id":"Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Distributed Training","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Getting Started","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/ONNX","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference","sidebar":"docs"},{"id":"Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/HyperOpt","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt","sidebar":"docs"},{"id":"Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview","sidebar":"docs"},{"id":"Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","path":"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Langchain","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/OpenAI","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding","sidebar":"docs"},{"id":"Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms","sidebar":"docs"},{"id":"Explore Algorithms/OpenCV/Image Transformations","path":"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Cyber ML","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures","sidebar":"docs"},{"id":"Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Data Cleaning","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Train Regressor","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor","sidebar":"docs"},{"id":"Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Data Balance Analysis","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Explanation Dashboard","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Image Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Interpreting Model Predictions","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/PDP and ICE Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Tabular Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Responsible AI/Text Explainers","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Contextual Bandits","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Multi-class classification","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Overview","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format","sidebar":"docs"},{"id":"Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","path":"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression","sidebar":"docs"},{"id":"Get Started/Create a Spark Cluster","path":"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster","sidebar":"docs"},{"id":"Get Started/Install SynapseML","path":"/SynapseML/docs/0.11.3/Get Started/Install SynapseML","sidebar":"docs"},{"id":"Get Started/Quickstart - Your First Models","path":"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models","sidebar":"docs"},{"id":"Get Started/Set up Cognitive Services","path":"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services","sidebar":"docs"},{"id":"Overview","path":"/SynapseML/docs/0.11.3/Overview","sidebar":"docs"},{"id":"Quick Examples/estimators/estimators_causal","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal"},{"id":"Quick Examples/estimators/estimators_cognitive","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive"},{"id":"Quick Examples/estimators/estimators_core","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core"},{"id":"Quick Examples/estimators/estimators_lightgbm","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm"},{"id":"Quick Examples/estimators/estimators_vw","path":"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw"},{"id":"Quick Examples/transformers/transformers_cognitive","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive"},{"id":"Quick Examples/transformers/transformers_core","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core"},{"id":"Quick Examples/transformers/transformers_deep_learning","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning"},{"id":"Quick Examples/transformers/transformers_opencv","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv"},{"id":"Quick Examples/transformers/transformers_vw","path":"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw"},{"id":"Reference/Contributor Guide","path":"/SynapseML/docs/0.11.3/Reference/Contributor Guide","sidebar":"docs"},{"id":"Reference/Developer Setup","path":"/SynapseML/docs/0.11.3/Reference/Developer Setup","sidebar":"docs"},{"id":"Reference/Docker Setup","path":"/SynapseML/docs/0.11.3/Reference/Docker Setup","sidebar":"docs"},{"id":"Reference/Dotnet Setup","path":"/SynapseML/docs/0.11.3/Reference/Dotnet Setup","sidebar":"docs"},{"id":"Reference/Quickstart - LightGBM in Dotnet","path":"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet","sidebar":"docs"},{"id":"Reference/R Setup","path":"/SynapseML/docs/0.11.3/Reference/R Setup","sidebar":"docs"},{"id":"Use with MLFlow/Autologging","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging","sidebar":"docs"},{"id":"Use with MLFlow/Install","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Install","sidebar":"docs"},{"id":"Use with MLFlow/Overview","path":"/SynapseML/docs/0.11.3/Use with MLFlow/Overview","sidebar":"docs"}],"draftIds":[],"sidebars":{"docs":{"link":{"path":"/SynapseML/docs/0.11.3/Overview","label":"What is SynapseML?"}}}}],"breadcrumbs":true}}}'),a=JSON.parse('{"defaultLocale":"en","locales":["en"],"path":"i18n","currentLocale":"en","localeConfigs":{"en":{"label":"English","direction":"ltr","htmlLang":"en","calendar":"gregory","path":"en"}}}');var i=n(57529);const c=JSON.parse('{"docusaurusVersion":"2.4.1","siteVersion":"0.0.0","pluginVersions":{"docusaurus-plugin-content-docs":{"type":"package","name":"@docusaurus/plugin-content-docs","version":"2.4.1"},"docusaurus-plugin-content-blog":{"type":"package","name":"@docusaurus/plugin-content-blog","version":"2.4.1"},"docusaurus-plugin-content-pages":{"type":"package","name":"@docusaurus/plugin-content-pages","version":"2.4.1"},"docusaurus-plugin-google-gtag":{"type":"package","name":"@docusaurus/plugin-google-gtag","version":"2.4.1"},"docusaurus-plugin-sitemap":{"type":"package","name":"@docusaurus/plugin-sitemap","version":"2.4.1"},"docusaurus-theme-classic":{"type":"package","name":"@docusaurus/theme-classic","version":"2.4.1"},"docusaurus-theme-search-algolia":{"type":"package","name":"@docusaurus/theme-search-algolia","version":"2.4.1"},"docusaurus-plugin-client-redirects":{"type":"package","name":"@docusaurus/plugin-client-redirects","version":"2.4.1"}}}'),l={siteConfig:r.default,siteMetadata:c,globalData:o,i18n:a,codeTranslations:i},d=s.createContext(l);function p(e){let{children:t}=e;return s.createElement(d.Provider,{value:l},t)}},44763:(e,t,n)=>{"use strict";n.d(t,{Z:()=>u});var s=n(67294),r=n(10412),o=n(35742),a=n(18780),i=n(7452);function c(e){let{error:t,tryAgain:n}=e;return s.createElement("div",{style:{display:"flex",flexDirection:"column",justifyContent:"center",alignItems:"flex-start",minHeight:"100vh",width:"100%",maxWidth:"80ch",fontSize:"20px",margin:"0 auto",padding:"1rem"}},s.createElement("h1",{style:{fontSize:"3rem"}},"This page crashed"),s.createElement("button",{type:"button",onClick:n,style:{margin:"1rem 0",fontSize:"2rem",cursor:"pointer",borderRadius:20,padding:"1rem"}},"Try again"),s.createElement(l,{error:t}))}function l(e){let{error:t}=e;const n=(0,a.getErrorCausalChain)(t).map((e=>e.message)).join("\n\nCause:\n");return s.createElement("p",{style:{whiteSpace:"pre-wrap"}},n)}function d(e){let{error:t,tryAgain:n}=e;return s.createElement(u,{fallback:()=>s.createElement(c,{error:t,tryAgain:n})},s.createElement(o.Z,null,s.createElement("title",null,"Page Error")),s.createElement(i.Z,null,s.createElement(c,{error:t,tryAgain:n})))}const p=e=>s.createElement(d,e);class u extends s.Component{constructor(e){super(e),this.state={error:null}}componentDidCatch(e){r.Z.canUseDOM&&this.setState({error:e})}render(){const{children:e}=this.props,{error:t}=this.state;if(t){var n;const e={error:t,tryAgain:()=>this.setState({error:null})};return(null!=(n=this.props.fallback)?n:p)(e)}return null!=e?e:null}}},10412:(e,t,n)=>{"use strict";n.d(t,{Z:()=>r});const s="undefined"!=typeof window&&"document"in window&&"createElement"in window.document,r={canUseDOM:s,canUseEventListeners:s&&("addEventListener"in window||"attachEvent"in window),canUseIntersectionObserver:s&&"IntersectionObserver"in window,canUseViewport:s&&"screen"in window}},35742:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(70405);function o(e){return s.createElement(r.ql,e)}},39960:(e,t,n)=>{"use strict";n.d(t,{Z:()=>m});var s=n(83117),r=n(67294),o=n(73727),a=n(18780),i=n(52263),c=n(13919),l=n(10412);const d=r.createContext({collectLink:()=>{}});var p=n(44996);function u(e,t){var n,u;let{isNavLink:m,to:h,href:g,activeClassName:f,isActive:b,"data-noBrokenLinkCheck":y,autoAddBaseUrl:x=!0,...v}=e;const{siteConfig:{trailingSlash:A,baseUrl:S}}=(0,i.Z)(),{withBaseUrl:E}=(0,p.C)(),L=(0,r.useContext)(d),M=(0,r.useRef)(null);(0,r.useImperativeHandle)(t,(()=>M.current));const k=h||g;const _=(0,c.Z)(k),w=null==k?void 0:k.replace("pathname://","");let I=void 0!==w?(C=w,x&&(e=>e.startsWith("/"))(C)?E(C):C):void 0;var C;I&&_&&(I=(0,a.applyTrailingSlash)(I,{trailingSlash:A,baseUrl:S}));const Q=(0,r.useRef)(!1),R=m?o.OL:o.rU,D=l.Z.canUseIntersectionObserver,T=(0,r.useRef)(),O=()=>{Q.current||null==I||(window.docusaurus.preload(I),Q.current=!0)};(0,r.useEffect)((()=>(!D&&_&&null!=I&&window.docusaurus.prefetch(I),()=>{D&&T.current&&T.current.disconnect()})),[T,I,D,_]);const P=null!=(n=null==(u=I)?void 0:u.startsWith("#"))&&n,F=!I||!_||P;return F||y||L.collectLink(I),F?r.createElement("a",(0,s.Z)({ref:M,href:I},k&&!_&&{target:"_blank",rel:"noopener noreferrer"},v)):r.createElement(R,(0,s.Z)({},v,{onMouseEnter:O,onTouchStart:O,innerRef:e=>{M.current=e,D&&e&&_&&(T.current=new window.IntersectionObserver((t=>{t.forEach((t=>{e===t.target&&(t.isIntersecting||t.intersectionRatio>0)&&(T.current.unobserve(e),T.current.disconnect(),null!=I&&window.docusaurus.prefetch(I))}))})),T.current.observe(e))},to:I},m&&{isActive:b,activeClassName:f}))}const m=r.forwardRef(u)},95999:(e,t,n)=>{"use strict";n.d(t,{Z:()=>c,I:()=>i});var s=n(67294);function r(e,t){const n=e.split(/(\{\w+\})/).map(((e,n)=>{if(n%2==1){const n=null==t?void 0:t[e.slice(1,-1)];if(void 0!==n)return n}return e}));return n.some((e=>(0,s.isValidElement)(e)))?n.map(((e,t)=>(0,s.isValidElement)(e)?s.cloneElement(e,{key:t}):e)).filter((e=>""!==e)):n.join("")}var o=n(57529);function a(e){var t,n;let{id:s,message:r}=e;if(void 0===s&&void 0===r)throw new Error("Docusaurus translation declarations must have at least a translation id or a default translation message");return null!=(t=null!=(n=o[null!=s?s:r])?n:r)?t:s}function i(e,t){let{message:n,id:s}=e;return r(a({message:n,id:s}),t)}function c(e){let{children:t,id:n,values:o}=e;if(t&&"string"!=typeof t)throw console.warn("Illegal <Translate> children",t),new Error("The Docusaurus <Translate> component only accept simple string values");const i=a({message:t,id:n});return s.createElement(s.Fragment,null,r(i,o))}},29935:(e,t,n)=>{"use strict";n.d(t,{m:()=>s});const s="default"},13919:(e,t,n)=>{"use strict";function s(e){return/^(?:\w*:|\/\/)/.test(e)}function r(e){return void 0!==e&&!s(e)}n.d(t,{Z:()=>r,b:()=>s})},44996:(e,t,n)=>{"use strict";n.d(t,{C:()=>a,Z:()=>i});var s=n(67294),r=n(52263),o=n(13919);function a(){const{siteConfig:{baseUrl:e,url:t}}=(0,r.Z)(),n=(0,s.useCallback)(((n,s)=>function(e,t,n,s){let{forcePrependBaseUrl:r=!1,absolute:a=!1}=void 0===s?{}:s;if(!n||n.startsWith("#")||(0,o.b)(n))return n;if(r)return t+n.replace(/^\//,"");if(n===t.replace(/\/$/,""))return t;const i=n.startsWith(t)?n:t+n.replace(/^\//,"");return a?e+i:i}(t,e,n,s)),[t,e]);return{withBaseUrl:n}}function i(e,t){void 0===t&&(t={});const{withBaseUrl:n}=a();return n(e,t)}},52263:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(58940);function o(){return(0,s.useContext)(r._)}},72389:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(98934);function o(){return(0,s.useContext)(r._)}},99670:(e,t,n)=>{"use strict";n.d(t,{Z:()=>s});function s(e){const t={};return function e(n,s){Object.entries(n).forEach((n=>{let[r,o]=n;const a=s?s+"."+r:r;var i;"object"==typeof(i=o)&&i&&Object.keys(i).length>0?e(o,a):t[a]=o}))}(e),t}},30226:(e,t,n)=>{"use strict";n.d(t,{_:()=>r,z:()=>o});var s=n(67294);const r=s.createContext(null);function o(e){let{children:t,value:n}=e;const o=s.useContext(r),a=(0,s.useMemo)((()=>function(e){let{parent:t,value:n}=e;if(!t){if(!n)throw new Error("Unexpected: no Docusaurus route context found");if(!("plugin"in n))throw new Error("Unexpected: Docusaurus topmost route context has no `plugin` attribute");return n}const s={...t.data,...null==n?void 0:n.data};return{plugin:t.plugin,data:s}}({parent:o,value:n})),[o,n]);return s.createElement(r.Provider,{value:a},t)}},80143:(e,t,n)=>{"use strict";n.d(t,{Iw:()=>b,gA:()=>m,WS:()=>h,_r:()=>p,Jo:()=>y,zh:()=>u,yW:()=>f,gB:()=>g});var s=n(16550),r=n(52263),o=n(29935);function a(e,t){void 0===t&&(t={});const n=function(){const{globalData:e}=(0,r.Z)();return e}()[e];if(!n&&t.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin.');return n}const i=e=>e.versions.find((e=>e.isLast));function c(e,t){const n=i(e);return[...e.versions.filter((e=>e!==n)),n].find((e=>!!(0,s.LX)(t,{path:e.path,exact:!1,strict:!1})))}function l(e,t){const n=c(e,t),r=null==n?void 0:n.docs.find((e=>!!(0,s.LX)(t,{path:e.path,exact:!0,strict:!1})));return{activeVersion:n,activeDoc:r,alternateDocVersions:r?function(t){const n={};return e.versions.forEach((e=>{e.docs.forEach((s=>{s.id===t&&(n[e.name]=s)}))})),n}(r.id):{}}}const d={},p=()=>{var e;return null!=(e=a("docusaurus-plugin-content-docs"))?e:d},u=e=>function(e,t,n){void 0===t&&(t=o.m),void 0===n&&(n={});const s=a(e),r=null==s?void 0:s[t];if(!r&&n.failfast)throw new Error('Docusaurus plugin global data not found for "'+e+'" plugin with id "'+t+'".');return r}("docusaurus-plugin-content-docs",e,{failfast:!0});function m(e){void 0===e&&(e={});const t=p(),{pathname:n}=(0,s.TH)();return function(e,t,n){void 0===n&&(n={});const r=Object.entries(e).sort(((e,t)=>t[1].path.localeCompare(e[1].path))).find((e=>{let[,n]=e;return!!(0,s.LX)(t,{path:n.path,exact:!1,strict:!1})})),o=r?{pluginId:r[0],pluginData:r[1]}:void 0;if(!o&&n.failfast)throw new Error("Can't find active docs plugin for \""+t+'" pathname, while it was expected to be found. Maybe you tried to use a docs feature that can only be used on a docs-related page? Existing docs plugin paths are: '+Object.values(e).map((e=>e.path)).join(", "));return o}(t,n,e)}function h(e){void 0===e&&(e={});const t=m(e),{pathname:n}=(0,s.TH)();if(!t)return;return{activePlugin:t,activeVersion:c(t.pluginData,n)}}function g(e){return u(e).versions}function f(e){const t=u(e);return i(t)}function b(e){const t=u(e),{pathname:n}=(0,s.TH)();return l(t,n)}function y(e){const t=u(e),{pathname:n}=(0,s.TH)();return function(e,t){const n=i(e);return{latestDocSuggestion:l(e,t).alternateDocVersions[n.name],latestVersionSuggestion:n}}(t,n)}},56657:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>s});const s={onRouteDidUpdate(e){let{location:t,previousLocation:n}=e;!n||t.pathname===n.pathname&&t.search===n.search&&t.hash===n.hash||setTimeout((()=>{window.gtag("event","page_view",{page_title:document.title,page_location:window.location.href,page_path:t.pathname+t.search+t.hash})}))}}},18320:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>o});var s=n(74865),r=n.n(s);r().configure({showSpinner:!1});const o={onRouteUpdate(e){let{location:t,previousLocation:n}=e;if(n&&t.pathname!==n.pathname){const e=window.setTimeout((()=>{r().start()}),200);return()=>window.clearTimeout(e)}},onRouteDidUpdate(){r().done()}}},3310:(e,t,n)=>{"use strict";n.r(t);var s=n(87410),r=n(36809);!function(e){const{themeConfig:{prism:t}}=r.default,{additionalLanguages:s}=t;globalThis.Prism=e,s.forEach((e=>{n(55979)("./prism-"+e)})),delete globalThis.Prism}(s.Z)},39471:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294);const r="iconExternalLink_nPIU";function o(e){let{width:t=13.5,height:n=13.5}=e;return s.createElement("svg",{width:t,height:n,"aria-hidden":"true",viewBox:"0 0 24 24",className:r},s.createElement("path",{fill:"currentColor",d:"M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"}))}},7452:(e,t,n)=>{"use strict";n.d(t,{Z:()=>Dt});var s=n(67294),r=n(86010),o=n(44763),a=n(10833),i=n(83117),c=n(16550),l=n(95999),d=n(85936);const p="__docusaurus_skipToContent_fallback";function u(e){e.setAttribute("tabindex","-1"),e.focus(),e.removeAttribute("tabindex")}function m(){const e=(0,s.useRef)(null),{action:t}=(0,c.k6)(),n=(0,s.useCallback)((e=>{e.preventDefault();const t=null!=(n=document.querySelector("main:first-of-type"))?n:document.getElementById(p);var n;t&&u(t)}),[]);return(0,d.S)((n=>{let{location:s}=n;e.current&&!s.hash&&"PUSH"===t&&u(e.current)})),{containerRef:e,onClick:n}}const h=(0,l.I)({id:"theme.common.skipToMainContent",description:"The skip to content label used for accessibility, allowing to rapidly navigate to main content with keyboard tab/enter navigation",message:"Skip to main content"});function g(e){var t;const n=null!=(t=e.children)?t:h,{containerRef:r,onClick:o}=m();return s.createElement("div",{ref:r,role:"region","aria-label":h},s.createElement("a",(0,i.Z)({},e,{href:"#"+p,onClick:o}),n))}var f=n(35281),b=n(19727);const y="skipToContent_fXgn";function x(){return s.createElement(g,{className:y})}var v=n(86668),A=n(59689);function S(e){let{width:t=21,height:n=21,color:r="currentColor",strokeWidth:o=1.2,className:a,...c}=e;return s.createElement("svg",(0,i.Z)({viewBox:"0 0 15 15",width:t,height:n},c),s.createElement("g",{stroke:r,strokeWidth:o},s.createElement("path",{d:"M.75.75l13.5 13.5M14.25.75L.75 14.25"})))}const E="closeButton_CVFx";function L(e){return s.createElement("button",(0,i.Z)({type:"button","aria-label":(0,l.I)({id:"theme.AnnouncementBar.closeButtonAriaLabel",message:"Close",description:"The ARIA label for close button of announcement bar"})},e,{className:(0,r.Z)("clean-btn close",E,e.className)}),s.createElement(S,{width:14,height:14,strokeWidth:3.1}))}const M="content_knG7";function k(e){const{announcementBar:t}=(0,v.L)(),{content:n}=t;return s.createElement("div",(0,i.Z)({},e,{className:(0,r.Z)(M,e.className),dangerouslySetInnerHTML:{__html:n}}))}const _="announcementBar_mb4j",w="announcementBarPlaceholder_vyr4",I="announcementBarClose_gvF7",C="announcementBarContent_xLdY";function Q(){const{announcementBar:e}=(0,v.L)(),{isActive:t,close:n}=(0,A.nT)();if(!t)return null;const{backgroundColor:r,textColor:o,isCloseable:a}=e;return s.createElement("div",{className:_,style:{backgroundColor:r,color:o},role:"banner"},a&&s.createElement("div",{className:w}),s.createElement(k,{className:C}),a&&s.createElement(L,{onClick:n,className:I}))}var R=n(93163),D=n(12466);var T=n(902),O=n(13102);const P=s.createContext(null);function F(e){let{children:t}=e;const n=function(){const e=(0,R.e)(),t=(0,O.HY)(),[n,r]=(0,s.useState)(!1),o=null!==t.component,a=(0,T.D9)(o);return(0,s.useEffect)((()=>{o&&!a&&r(!0)}),[o,a]),(0,s.useEffect)((()=>{o?e.shown||r(!0):r(!1)}),[e.shown,o]),(0,s.useMemo)((()=>[n,r]),[n])}();return s.createElement(P.Provider,{value:n},t)}function N(e){if(e.component){const t=e.component;return s.createElement(t,e.props)}}function B(){const e=(0,s.useContext)(P);if(!e)throw new T.i6("NavbarSecondaryMenuDisplayProvider");const[t,n]=e,r=(0,s.useCallback)((()=>n(!1)),[n]),o=(0,O.HY)();return(0,s.useMemo)((()=>({shown:t,hide:r,content:N(o)})),[r,o,t])}function V(e){let{header:t,primaryMenu:n,secondaryMenu:o}=e;const{shown:a}=B();return s.createElement("div",{className:"navbar-sidebar"},t,s.createElement("div",{className:(0,r.Z)("navbar-sidebar__items",{"navbar-sidebar__items--show-secondary":a})},s.createElement("div",{className:"navbar-sidebar__item menu"},n),s.createElement("div",{className:"navbar-sidebar__item menu"},o)))}var G=n(92949),W=n(72389);function U(e){return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:24,height:24},e),s.createElement("path",{fill:"currentColor",d:"M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"}))}function z(e){return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:24,height:24},e),s.createElement("path",{fill:"currentColor",d:"M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"}))}const H={toggle:"toggle_vylO",toggleButton:"toggleButton_gllP",darkToggleIcon:"darkToggleIcon_wfgR",lightToggleIcon:"lightToggleIcon_pyhR",toggleButtonDisabled:"toggleButtonDisabled_aARS"};function j(e){let{className:t,buttonClassName:n,value:o,onChange:a}=e;const i=(0,W.Z)(),c=(0,l.I)({message:"Switch between dark and light mode (currently {mode})",id:"theme.colorToggle.ariaLabel",description:"The ARIA label for the navbar color mode toggle"},{mode:"dark"===o?(0,l.I)({message:"dark mode",id:"theme.colorToggle.ariaLabel.mode.dark",description:"The name for the dark color mode"}):(0,l.I)({message:"light mode",id:"theme.colorToggle.ariaLabel.mode.light",description:"The name for the light color mode"})});return s.createElement("div",{className:(0,r.Z)(H.toggle,t)},s.createElement("button",{className:(0,r.Z)("clean-btn",H.toggleButton,!i&&H.toggleButtonDisabled,n),type:"button",onClick:()=>a("dark"===o?"light":"dark"),disabled:!i,title:c,"aria-label":c,"aria-live":"polite"},s.createElement(U,{className:(0,r.Z)(H.toggleIcon,H.lightToggleIcon)}),s.createElement(z,{className:(0,r.Z)(H.toggleIcon,H.darkToggleIcon)})))}const $=s.memo(j),Z="darkNavbarColorModeToggle_X3D1";function X(e){let{className:t}=e;const n=(0,v.L)().navbar.style,r=(0,v.L)().colorMode.disableSwitch,{colorMode:o,setColorMode:a}=(0,G.I)();return r?null:s.createElement($,{className:t,buttonClassName:"dark"===n?Z:void 0,value:o,onChange:a})}var K=n(21327);function Y(){return s.createElement(K.Z,{className:"navbar__brand",imageClassName:"navbar__logo",titleClassName:"navbar__title text--truncate"})}function q(){const e=(0,R.e)();return s.createElement("button",{type:"button","aria-label":(0,l.I)({id:"theme.docs.sidebar.closeSidebarButtonAriaLabel",message:"Close navigation bar",description:"The ARIA label for close button of mobile sidebar"}),className:"clean-btn navbar-sidebar__close",onClick:()=>e.toggle()},s.createElement(S,{color:"var(--ifm-color-emphasis-600)"}))}function J(){return s.createElement("div",{className:"navbar-sidebar__brand"},s.createElement(Y,null),s.createElement(X,{className:"margin-right--md"}),s.createElement(q,null))}var ee=n(39960),te=n(44996),ne=n(13919),se=n(98022),re=n(39471);function oe(e){let{activeBasePath:t,activeBaseRegex:n,to:r,href:o,label:a,html:c,isDropdownLink:l,prependBaseUrlToHref:d,...p}=e;const u=(0,te.Z)(r),m=(0,te.Z)(t),h=(0,te.Z)(o,{forcePrependBaseUrl:!0}),g=a&&o&&!(0,ne.Z)(o),f=c?{dangerouslySetInnerHTML:{__html:c}}:{children:s.createElement(s.Fragment,null,a,g&&s.createElement(re.Z,l&&{width:12,height:12}))};return o?s.createElement(ee.Z,(0,i.Z)({href:d?h:o},p,f)):s.createElement(ee.Z,(0,i.Z)({to:u,isNavLink:!0},(t||n)&&{isActive:(e,t)=>n?(0,se.F)(n,t.pathname):t.pathname.startsWith(m)},p,f))}function ae(e){let{className:t,isDropdownItem:n=!1,...o}=e;const a=s.createElement(oe,(0,i.Z)({className:(0,r.Z)(n?"dropdown__link":"navbar__item navbar__link",t),isDropdownLink:n},o));return n?s.createElement("li",null,a):a}function ie(e){let{className:t,isDropdownItem:n,...o}=e;return s.createElement("li",{className:"menu__list-item"},s.createElement(oe,(0,i.Z)({className:(0,r.Z)("menu__link",t)},o)))}function ce(e){var t;let{mobile:n=!1,position:r,...o}=e;const a=n?ie:ae;return s.createElement(a,(0,i.Z)({},o,{activeClassName:null!=(t=o.activeClassName)?t:n?"menu__link--active":"navbar__link--active"}))}var le=n(86043),de=n(48596),pe=n(52263);function ue(e,t){return e.some((e=>function(e,t){return!!(0,de.Mg)(e.to,t)||!!(0,se.F)(e.activeBaseRegex,t)||!(!e.activeBasePath||!t.startsWith(e.activeBasePath))}(e,t)))}function me(e){var t;let{items:n,position:o,className:a,onClick:c,...l}=e;const d=(0,s.useRef)(null),[p,u]=(0,s.useState)(!1);return(0,s.useEffect)((()=>{const e=e=>{d.current&&!d.current.contains(e.target)&&u(!1)};return document.addEventListener("mousedown",e),document.addEventListener("touchstart",e),document.addEventListener("focusin",e),()=>{document.removeEventListener("mousedown",e),document.removeEventListener("touchstart",e),document.removeEventListener("focusin",e)}}),[d]),s.createElement("div",{ref:d,className:(0,r.Z)("navbar__item","dropdown","dropdown--hoverable",{"dropdown--right":"right"===o,"dropdown--show":p})},s.createElement(oe,(0,i.Z)({"aria-haspopup":"true","aria-expanded":p,role:"button",href:l.to?void 0:"#",className:(0,r.Z)("navbar__link",a)},l,{onClick:l.to?void 0:e=>e.preventDefault(),onKeyDown:e=>{"Enter"===e.key&&(e.preventDefault(),u(!p))}}),null!=(t=l.children)?t:l.label),s.createElement("ul",{className:"dropdown__menu"},n.map(((e,t)=>s.createElement(je,(0,i.Z)({isDropdownItem:!0,activeClassName:"dropdown__link--active"},e,{key:t}))))))}function he(e){var t;let{items:n,className:o,position:a,onClick:l,...d}=e;const p=function(){const{siteConfig:{baseUrl:e}}=(0,pe.Z)(),{pathname:t}=(0,c.TH)();return t.replace(e,"/")}(),u=ue(n,p),{collapsed:m,toggleCollapsed:h,setCollapsed:g}=(0,le.u)({initialState:()=>!u});return(0,s.useEffect)((()=>{u&&g(!u)}),[p,u,g]),s.createElement("li",{className:(0,r.Z)("menu__list-item",{"menu__list-item--collapsed":m})},s.createElement(oe,(0,i.Z)({role:"button",className:(0,r.Z)("menu__link menu__link--sublist menu__link--sublist-caret",o)},d,{onClick:e=>{e.preventDefault(),h()}}),null!=(t=d.children)?t:d.label),s.createElement(le.z,{lazy:!0,as:"ul",className:"menu__list",collapsed:m},n.map(((e,t)=>s.createElement(je,(0,i.Z)({mobile:!0,isDropdownItem:!0,onClick:l,activeClassName:"menu__link--active"},e,{key:t}))))))}function ge(e){let{mobile:t=!1,...n}=e;const r=t?he:me;return s.createElement(r,n)}var fe=n(94711);function be(e){let{width:t=20,height:n=20,...r}=e;return s.createElement("svg",(0,i.Z)({viewBox:"0 0 24 24",width:t,height:n,"aria-hidden":!0},r),s.createElement("path",{fill:"currentColor",d:"M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"}))}const ye="iconLanguage_nlXk";function xe(){return s.createElement("svg",{width:"15",height:"15",className:"DocSearch-Control-Key-Icon"},s.createElement("path",{d:"M4.505 4.496h2M5.505 5.496v5M8.216 4.496l.055 5.993M10 7.5c.333.333.5.667.5 1v2M12.326 4.5v5.996M8.384 4.496c1.674 0 2.116 0 2.116 1.5s-.442 1.5-2.116 1.5M3.205 9.303c-.09.448-.277 1.21-1.241 1.203C1 10.5.5 9.513.5 8V7c0-1.57.5-2.5 1.464-2.494.964.006 1.134.598 1.24 1.342M12.553 10.5h1.953",strokeWidth:"1.2",stroke:"currentColor",fill:"none",strokeLinecap:"square"}))}var ve=n(20830),Ae=["translations"];function Se(){return Se=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},Se.apply(this,arguments)}function Ee(e,t){return function(e){if(Array.isArray(e))return e}(e)||function(e,t){var n=null==e?null:"undefined"!=typeof Symbol&&e[Symbol.iterator]||e["@@iterator"];if(null==n)return;var s,r,o=[],a=!0,i=!1;try{for(n=n.call(e);!(a=(s=n.next()).done)&&(o.push(s.value),!t||o.length!==t);a=!0);}catch(c){i=!0,r=c}finally{try{a||null==n.return||n.return()}finally{if(i)throw r}}return o}(e,t)||function(e,t){if(!e)return;if("string"==typeof e)return Le(e,t);var n=Object.prototype.toString.call(e).slice(8,-1);"Object"===n&&e.constructor&&(n=e.constructor.name);if("Map"===n||"Set"===n)return Array.from(e);if("Arguments"===n||/^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n))return Le(e,t)}(e,t)||function(){throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.")}()}function Le(e,t){(null==t||t>e.length)&&(t=e.length);for(var n=0,s=new Array(t);n<t;n++)s[n]=e[n];return s}function Me(e,t){if(null==e)return{};var n,s,r=function(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||(r[n]=e[n]);return r}(e,t);if(Object.getOwnPropertySymbols){var o=Object.getOwnPropertySymbols(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||Object.prototype.propertyIsEnumerable.call(e,n)&&(r[n]=e[n])}return r}var ke="Ctrl";var _e=s.forwardRef((function(e,t){var n=e.translations,r=void 0===n?{}:n,o=Me(e,Ae),a=r.buttonText,i=void 0===a?"Search":a,c=r.buttonAriaLabel,l=void 0===c?"Search":c,d=Ee((0,s.useState)(null),2),p=d[0],u=d[1];return(0,s.useEffect)((function(){"undefined"!=typeof navigator&&(/(Mac|iPhone|iPod|iPad)/i.test(navigator.platform)?u("\u2318"):u(ke))}),[]),s.createElement("button",Se({type:"button",className:"DocSearch DocSearch-Button","aria-label":l},o,{ref:t}),s.createElement("span",{className:"DocSearch-Button-Container"},s.createElement(ve.W,null),s.createElement("span",{className:"DocSearch-Button-Placeholder"},i)),s.createElement("span",{className:"DocSearch-Button-Keys"},null!==p&&s.createElement(s.Fragment,null,s.createElement("kbd",{className:"DocSearch-Button-Key"},p===ke?s.createElement(xe,null):p),s.createElement("kbd",{className:"DocSearch-Button-Key"},"K"))))})),we=n(35742),Ie=n(66177),Ce=n(239),Qe=n(43320);var Re=n(73935);const De={button:{buttonText:(0,l.I)({id:"theme.SearchBar.label",message:"Search",description:"The ARIA label and placeholder for search button"}),buttonAriaLabel:(0,l.I)({id:"theme.SearchBar.label",message:"Search",description:"The ARIA label and placeholder for search button"})},modal:{searchBox:{resetButtonTitle:(0,l.I)({id:"theme.SearchModal.searchBox.resetButtonTitle",message:"Clear the query",description:"The label and ARIA label for search box reset button"}),resetButtonAriaLabel:(0,l.I)({id:"theme.SearchModal.searchBox.resetButtonTitle",message:"Clear the query",description:"The label and ARIA label for search box reset button"}),cancelButtonText:(0,l.I)({id:"theme.SearchModal.searchBox.cancelButtonText",message:"Cancel",description:"The label and ARIA label for search box cancel button"}),cancelButtonAriaLabel:(0,l.I)({id:"theme.SearchModal.searchBox.cancelButtonText",message:"Cancel",description:"The label and ARIA label for search box cancel button"})},startScreen:{recentSearchesTitle:(0,l.I)({id:"theme.SearchModal.startScreen.recentSearchesTitle",message:"Recent",description:"The title for recent searches"}),noRecentSearchesText:(0,l.I)({id:"theme.SearchModal.startScreen.noRecentSearchesText",message:"No recent searches",description:"The text when no recent searches"}),saveRecentSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.saveRecentSearchButtonTitle",message:"Save this search",description:"The label for save recent search button"}),removeRecentSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.removeRecentSearchButtonTitle",message:"Remove this search from history",description:"The label for remove recent search button"}),favoriteSearchesTitle:(0,l.I)({id:"theme.SearchModal.startScreen.favoriteSearchesTitle",message:"Favorite",description:"The title for favorite searches"}),removeFavoriteSearchButtonTitle:(0,l.I)({id:"theme.SearchModal.startScreen.removeFavoriteSearchButtonTitle",message:"Remove this search from favorites",description:"The label for remove favorite search button"})},errorScreen:{titleText:(0,l.I)({id:"theme.SearchModal.errorScreen.titleText",message:"Unable to fetch results",description:"The title for error screen of search modal"}),helpText:(0,l.I)({id:"theme.SearchModal.errorScreen.helpText",message:"You might want to check your network connection.",description:"The help text for error screen of search modal"})},footer:{selectText:(0,l.I)({id:"theme.SearchModal.footer.selectText",message:"to select",description:"The explanatory text of the action for the enter key"}),selectKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.selectKeyAriaLabel",message:"Enter key",description:"The ARIA label for the Enter key button that makes the selection"}),navigateText:(0,l.I)({id:"theme.SearchModal.footer.navigateText",message:"to navigate",description:"The explanatory text of the action for the Arrow up and Arrow down key"}),navigateUpKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.navigateUpKeyAriaLabel",message:"Arrow up",description:"The ARIA label for the Arrow up key button that makes the navigation"}),navigateDownKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.navigateDownKeyAriaLabel",message:"Arrow down",description:"The ARIA label for the Arrow down key button that makes the navigation"}),closeText:(0,l.I)({id:"theme.SearchModal.footer.closeText",message:"to close",description:"The explanatory text of the action for Escape key"}),closeKeyAriaLabel:(0,l.I)({id:"theme.SearchModal.footer.closeKeyAriaLabel",message:"Escape key",description:"The ARIA label for the Escape key button that close the modal"}),searchByText:(0,l.I)({id:"theme.SearchModal.footer.searchByText",message:"Search by",description:"The text explain that the search is making by Algolia"})},noResultsScreen:{noResultsText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.noResultsText",message:"No results for",description:"The text explains that there are no results for the following search"}),suggestedQueryText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.suggestedQueryText",message:"Try searching for",description:"The text for the suggested query when no results are found for the following search"}),reportMissingResultsText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.reportMissingResultsText",message:"Believe this query should return results?",description:"The text for the question where the user thinks there are missing results"}),reportMissingResultsLinkText:(0,l.I)({id:"theme.SearchModal.noResultsScreen.reportMissingResultsLinkText",message:"Let us know.",description:"The text for the link to report missing results"})}},placeholder:(0,l.I)({id:"theme.SearchModal.placeholder",message:"Search docs",description:"The placeholder of the input of the DocSearch pop-up modal"})};let Te=null;function Oe(e){let{hit:t,children:n}=e;return s.createElement(ee.Z,{to:t.url},n)}function Pe(e){let{state:t,onClose:n}=e;const r=(0,Ie.M)();return s.createElement(ee.Z,{to:r(t.query),onClick:n},s.createElement(l.Z,{id:"theme.SearchBar.seeAll",values:{count:t.context.nbHits}},"See all {count} results"))}function Fe(e){var t,r;let{contextualSearch:o,externalUrlRegex:a,...l}=e;const{siteMetadata:d}=(0,pe.Z)(),p=(0,Ce.l)(),u=function(){const{locale:e,tags:t}=(0,Qe._q)();return["language:"+e,t.map((e=>"docusaurus_tag:"+e))]}(),m=null!=(t=null==(r=l.searchParameters)?void 0:r.facetFilters)?t:[],h=o?function(e,t){const n=e=>"string"==typeof e?[e]:e;return[...n(e),...n(t)]}(u,m):m,g={...l.searchParameters,facetFilters:h},f=(0,c.k6)(),b=(0,s.useRef)(null),y=(0,s.useRef)(null),[x,v]=(0,s.useState)(!1),[A,S]=(0,s.useState)(void 0),E=(0,s.useCallback)((()=>Te?Promise.resolve():Promise.all([n.e(76780).then(n.bind(n,76780)),Promise.all([n.e(40532),n.e(46945)]).then(n.bind(n,46945)),Promise.all([n.e(40532),n.e(18894)]).then(n.bind(n,18894))]).then((e=>{let[{DocSearchModal:t}]=e;Te=t}))),[]),L=(0,s.useCallback)((()=>{E().then((()=>{b.current=document.createElement("div"),document.body.insertBefore(b.current,document.body.firstChild),v(!0)}))}),[E,v]),M=(0,s.useCallback)((()=>{var e;v(!1),null==(e=b.current)||e.remove()}),[v]),k=(0,s.useCallback)((e=>{E().then((()=>{v(!0),S(e.key)}))}),[E,v,S]),_=(0,s.useRef)({navigate(e){let{itemUrl:t}=e;(0,se.F)(a,t)?window.location.href=t:f.push(t)}}).current,w=(0,s.useRef)((e=>l.transformItems?l.transformItems(e):e.map((e=>({...e,url:p(e.url)}))))).current,I=(0,s.useMemo)((()=>e=>s.createElement(Pe,(0,i.Z)({},e,{onClose:M}))),[M]),C=(0,s.useCallback)((e=>(e.addAlgoliaAgent("docusaurus",d.docusaurusVersion),e)),[d.docusaurusVersion]);return function(e){var t=e.isOpen,n=e.onOpen,r=e.onClose,o=e.onInput,a=e.searchButtonRef;s.useEffect((function(){function e(e){(27===e.keyCode&&t||"k"===e.key.toLowerCase()&&(e.metaKey||e.ctrlKey)||!function(e){var t=e.target,n=t.tagName;return t.isContentEditable||"INPUT"===n||"SELECT"===n||"TEXTAREA"===n}(e)&&"/"===e.key&&!t)&&(e.preventDefault(),t?r():document.body.classList.contains("DocSearch--active")||document.body.classList.contains("DocSearch--active")||n()),a&&a.current===document.activeElement&&o&&/[a-zA-Z0-9]/.test(String.fromCharCode(e.keyCode))&&o(e)}return window.addEventListener("keydown",e),function(){window.removeEventListener("keydown",e)}}),[t,n,r,o,a])}({isOpen:x,onOpen:L,onClose:M,onInput:k,searchButtonRef:y}),s.createElement(s.Fragment,null,s.createElement(we.Z,null,s.createElement("link",{rel:"preconnect",href:"https://"+l.appId+"-dsn.algolia.net",crossOrigin:"anonymous"})),s.createElement(_e,{onTouchStart:E,onFocus:E,onMouseOver:E,onClick:L,ref:y,translations:De.button}),x&&Te&&b.current&&(0,Re.createPortal)(s.createElement(Te,(0,i.Z)({onClose:M,initialScrollY:window.scrollY,initialQuery:A,navigator:_,transformItems:w,hitComponent:Oe,transformSearchClient:C},l.searchPagePath&&{resultsFooterComponent:I},l,{searchParameters:g,placeholder:De.placeholder,translations:De.modal})),b.current))}function Ne(){const{siteConfig:e}=(0,pe.Z)();return s.createElement(Fe,e.themeConfig.algolia)}const Be="searchBox_ZlJk";function Ve(e){let{children:t,className:n}=e;return s.createElement("div",{className:(0,r.Z)(n,Be)},t)}var Ge=n(80143),We=n(53438);var Ue=n(60373);const ze=e=>e.docs.find((t=>t.id===e.mainDocId));const He={default:ce,localeDropdown:function(e){let{mobile:t,dropdownItemsBefore:n,dropdownItemsAfter:r,...o}=e;const{i18n:{currentLocale:a,locales:d,localeConfigs:p}}=(0,pe.Z)(),u=(0,fe.l)(),{search:m,hash:h}=(0,c.TH)(),g=[...n,...d.map((e=>{const n=""+("pathname://"+u.createUrl({locale:e,fullyQualified:!1}))+m+h;return{label:p[e].label,lang:p[e].htmlLang,to:n,target:"_self",autoAddBaseUrl:!1,className:e===a?t?"menu__link--active":"dropdown__link--active":""}})),...r],f=t?(0,l.I)({message:"Languages",id:"theme.navbar.mobileLanguageDropdown.label",description:"The label for the mobile language switcher dropdown"}):p[a].label;return s.createElement(ge,(0,i.Z)({},o,{mobile:t,label:s.createElement(s.Fragment,null,s.createElement(be,{className:ye}),f),items:g}))},search:function(e){let{mobile:t,className:n}=e;return t?null:s.createElement(Ve,{className:n},s.createElement(Ne,null))},dropdown:ge,html:function(e){let{value:t,className:n,mobile:o=!1,isDropdownItem:a=!1}=e;const i=a?"li":"div";return s.createElement(i,{className:(0,r.Z)({navbar__item:!o&&!a,"menu__list-item":o},n),dangerouslySetInnerHTML:{__html:t}})},doc:function(e){let{docId:t,label:n,docsPluginId:r,...o}=e;const{activeDoc:a}=(0,Ge.Iw)(r),c=(0,We.vY)(t,r);return null===c?null:s.createElement(ce,(0,i.Z)({exact:!0},o,{isActive:()=>(null==a?void 0:a.path)===c.path||!(null==a||!a.sidebar)&&a.sidebar===c.sidebar,label:null!=n?n:c.id,to:c.path}))},docSidebar:function(e){let{sidebarId:t,label:n,docsPluginId:r,...o}=e;const{activeDoc:a}=(0,Ge.Iw)(r),c=(0,We.oz)(t,r).link;if(!c)throw new Error('DocSidebarNavbarItem: Sidebar with ID "'+t+"\" doesn't have anything to be linked to.");return s.createElement(ce,(0,i.Z)({exact:!0},o,{isActive:()=>(null==a?void 0:a.sidebar)===t,label:null!=n?n:c.label,to:c.path}))},docsVersion:function(e){let{label:t,to:n,docsPluginId:r,...o}=e;const a=(0,We.lO)(r)[0],c=null!=t?t:a.label,l=null!=n?n:(e=>e.docs.find((t=>t.id===e.mainDocId)))(a).path;return s.createElement(ce,(0,i.Z)({},o,{label:c,to:l}))},docsVersionDropdown:function(e){let{mobile:t,docsPluginId:n,dropdownActiveClassDisabled:r,dropdownItemsBefore:o,dropdownItemsAfter:a,...d}=e;const{search:p,hash:u}=(0,c.TH)(),m=(0,Ge.Iw)(n),h=(0,Ge.gB)(n),{savePreferredVersionName:g}=(0,Ue.J)(n),f=[...o,...h.map((e=>{var t;const n=null!=(t=m.alternateDocVersions[e.name])?t:ze(e);return{label:e.label,to:""+n.path+p+u,isActive:()=>e===m.activeVersion,onClick:()=>g(e.name)}})),...a],b=(0,We.lO)(n)[0],y=t&&f.length>1?(0,l.I)({id:"theme.navbar.mobileVersionsDropdown.label",message:"Versions",description:"The label for the navbar versions dropdown on mobile view"}):b.label,x=t&&f.length>1?void 0:ze(b).path;return f.length<=1?s.createElement(ce,(0,i.Z)({},d,{mobile:t,label:y,to:x,isActive:r?()=>!1:void 0})):s.createElement(ge,(0,i.Z)({},d,{mobile:t,label:y,to:x,items:f,isActive:r?()=>!1:void 0}))}};function je(e){let{type:t,...n}=e;const r=function(e,t){return e&&"default"!==e?e:"items"in t?"dropdown":"default"}(t,n),o=He[r];if(!o)throw new Error('No NavbarItem component found for type "'+t+'".');return s.createElement(o,n)}function $e(){const e=(0,R.e)(),t=(0,v.L)().navbar.items;return s.createElement("ul",{className:"menu__list"},t.map(((t,n)=>s.createElement(je,(0,i.Z)({mobile:!0},t,{onClick:()=>e.toggle(),key:n})))))}function Ze(e){return s.createElement("button",(0,i.Z)({},e,{type:"button",className:"clean-btn navbar-sidebar__back"}),s.createElement(l.Z,{id:"theme.navbar.mobileSidebarSecondaryMenu.backButtonLabel",description:"The label of the back button to return to main menu, inside the mobile navbar sidebar secondary menu (notably used to display the docs sidebar)"},"\u2190 Back to main menu"))}function Xe(){const e=0===(0,v.L)().navbar.items.length,t=B();return s.createElement(s.Fragment,null,!e&&s.createElement(Ze,{onClick:()=>t.hide()}),t.content)}function Ke(){const e=(0,R.e)();var t;return void 0===(t=e.shown)&&(t=!0),(0,s.useEffect)((()=>(document.body.style.overflow=t?"hidden":"visible",()=>{document.body.style.overflow="visible"})),[t]),e.shouldRender?s.createElement(V,{header:s.createElement(J,null),primaryMenu:s.createElement($e,null),secondaryMenu:s.createElement(Xe,null)}):null}const Ye="navbarHideable_m1mJ",qe="navbarHidden_jGov";function Je(e){return s.createElement("div",(0,i.Z)({role:"presentation"},e,{className:(0,r.Z)("navbar-sidebar__backdrop",e.className)}))}function et(e){let{children:t}=e;const{navbar:{hideOnScroll:n,style:o}}=(0,v.L)(),a=(0,R.e)(),{navbarRef:i,isNavbarVisible:c}=function(e){const[t,n]=(0,s.useState)(e),r=(0,s.useRef)(!1),o=(0,s.useRef)(0),a=(0,s.useCallback)((e=>{null!==e&&(o.current=e.getBoundingClientRect().height)}),[]);return(0,D.RF)(((t,s)=>{let{scrollY:a}=t;if(!e)return;if(a<o.current)return void n(!0);if(r.current)return void(r.current=!1);const i=null==s?void 0:s.scrollY,c=document.documentElement.scrollHeight-o.current,l=window.innerHeight;i&&a>=i?n(!1):a+l<c&&n(!0)})),(0,d.S)((t=>{if(!e)return;const s=t.location.hash;if(s?document.getElementById(s.substring(1)):void 0)return r.current=!0,void n(!1);n(!0)})),{navbarRef:a,isNavbarVisible:t}}(n);return s.createElement("nav",{ref:i,"aria-label":(0,l.I)({id:"theme.NavBar.navAriaLabel",message:"Main",description:"The ARIA label for the main navigation"}),className:(0,r.Z)("navbar","navbar--fixed-top",n&&[Ye,!c&&qe],{"navbar--dark":"dark"===o,"navbar--primary":"primary"===o,"navbar-sidebar--show":a.shown})},t,s.createElement(Je,{onClick:a.toggle}),s.createElement(Ke,null))}var tt=n(18780);const nt="errorBoundaryError_a6uf";function st(e){return s.createElement("button",(0,i.Z)({type:"button"},e),s.createElement(l.Z,{id:"theme.ErrorPageContent.tryAgain",description:"The label of the button to try again rendering when the React error boundary captures an error"},"Try again"))}function rt(e){let{error:t}=e;const n=(0,tt.getErrorCausalChain)(t).map((e=>e.message)).join("\n\nCause:\n");return s.createElement("p",{className:nt},n)}class ot extends s.Component{componentDidCatch(e,t){throw this.props.onError(e,t)}render(){return this.props.children}}function at(e){let{width:t=30,height:n=30,className:r,...o}=e;return s.createElement("svg",(0,i.Z)({className:r,width:t,height:n,viewBox:"0 0 30 30","aria-hidden":"true"},o),s.createElement("path",{stroke:"currentColor",strokeLinecap:"round",strokeMiterlimit:"10",strokeWidth:"2",d:"M4 7h22M4 15h22M4 23h22"}))}function it(){const{toggle:e,shown:t}=(0,R.e)();return s.createElement("button",{onClick:e,"aria-label":(0,l.I)({id:"theme.docs.sidebar.toggleSidebarButtonAriaLabel",message:"Toggle navigation bar",description:"The ARIA label for hamburger menu button of mobile navigation"}),"aria-expanded":t,className:"navbar__toggle clean-btn",type:"button"},s.createElement(at,null))}const ct="colorModeToggle_DEke";function lt(e){let{items:t}=e;return s.createElement(s.Fragment,null,t.map(((e,t)=>s.createElement(ot,{key:t,onError:t=>new Error("A theme navbar item failed to render.\nPlease double-check the following navbar item (themeConfig.navbar.items) of your Docusaurus config:\n"+JSON.stringify(e,null,2),{cause:t})},s.createElement(je,e)))))}function dt(e){let{left:t,right:n}=e;return s.createElement("div",{className:"navbar__inner"},s.createElement("div",{className:"navbar__items"},t),s.createElement("div",{className:"navbar__items navbar__items--right"},n))}function pt(){const e=(0,R.e)(),t=(0,v.L)().navbar.items,[n,r]=function(e){function t(e){var t;return"left"===(null!=(t=e.position)?t:"right")}return[e.filter(t),e.filter((e=>!t(e)))]}(t),o=t.find((e=>"search"===e.type));return s.createElement(dt,{left:s.createElement(s.Fragment,null,!e.disabled&&s.createElement(it,null),s.createElement(Y,null),s.createElement(lt,{items:n})),right:s.createElement(s.Fragment,null,s.createElement(lt,{items:r}),s.createElement(X,{className:ct}),!o&&s.createElement(Ve,null,s.createElement(Ne,null)))})}function ut(){return s.createElement(et,null,s.createElement(pt,null))}function mt(e){let{item:t}=e;const{to:n,href:r,label:o,prependBaseUrlToHref:a,...c}=t,l=(0,te.Z)(n),d=(0,te.Z)(r,{forcePrependBaseUrl:!0});return s.createElement(ee.Z,(0,i.Z)({className:"footer__link-item"},r?{href:a?d:r}:{to:l},c),o,r&&!(0,ne.Z)(r)&&s.createElement(re.Z,null))}function ht(e){var t;let{item:n}=e;return n.html?s.createElement("li",{className:"footer__item",dangerouslySetInnerHTML:{__html:n.html}}):s.createElement("li",{key:null!=(t=n.href)?t:n.to,className:"footer__item"},s.createElement(mt,{item:n}))}function gt(e){let{column:t}=e;return s.createElement("div",{className:"col footer__col"},s.createElement("div",{className:"footer__title"},t.title),s.createElement("ul",{className:"footer__items clean-list"},t.items.map(((e,t)=>s.createElement(ht,{key:t,item:e})))))}function ft(e){let{columns:t}=e;return s.createElement("div",{className:"row footer__links"},t.map(((e,t)=>s.createElement(gt,{key:t,column:e}))))}function bt(){return s.createElement("span",{className:"footer__link-separator"},"\xb7")}function yt(e){let{item:t}=e;return t.html?s.createElement("span",{className:"footer__link-item",dangerouslySetInnerHTML:{__html:t.html}}):s.createElement(mt,{item:t})}function xt(e){let{links:t}=e;return s.createElement("div",{className:"footer__links text--center"},s.createElement("div",{className:"footer__links"},t.map(((e,n)=>s.createElement(s.Fragment,{key:n},s.createElement(yt,{item:e}),t.length!==n+1&&s.createElement(bt,null))))))}function vt(e){let{links:t}=e;return function(e){return"title"in e[0]}(t)?s.createElement(ft,{columns:t}):s.createElement(xt,{links:t})}var At=n(50941);const St="footerLogoLink_BH7S";function Et(e){var t;let{logo:n}=e;const{withBaseUrl:o}=(0,te.C)(),a={light:o(n.src),dark:o(null!=(t=n.srcDark)?t:n.src)};return s.createElement(At.Z,{className:(0,r.Z)("footer__logo",n.className),alt:n.alt,sources:a,width:n.width,height:n.height,style:n.style})}function Lt(e){let{logo:t}=e;return t.href?s.createElement(ee.Z,{href:t.href,className:St,target:t.target},s.createElement(Et,{logo:t})):s.createElement(Et,{logo:t})}function Mt(e){let{copyright:t}=e;return s.createElement("div",{className:"footer__copyright",dangerouslySetInnerHTML:{__html:t}})}function kt(e){let{style:t,links:n,logo:o,copyright:a}=e;return s.createElement("footer",{className:(0,r.Z)("footer",{"footer--dark":"dark"===t})},s.createElement("div",{className:"container container-fluid"},n,(o||a)&&s.createElement("div",{className:"footer__bottom text--center"},o&&s.createElement("div",{className:"margin-bottom--sm"},o),a)))}function _t(){const{footer:e}=(0,v.L)();if(!e)return null;const{copyright:t,links:n,logo:r,style:o}=e;return s.createElement(kt,{style:o,links:n&&n.length>0&&s.createElement(vt,{links:n}),logo:r&&s.createElement(Lt,{logo:r}),copyright:t&&s.createElement(Mt,{copyright:t})})}const wt=s.memo(_t),It=(0,T.Qc)([G.S,A.pl,D.OC,Ue.L5,a.VC,function(e){let{children:t}=e;return s.createElement(O.n2,null,s.createElement(R.M,null,s.createElement(F,null,t)))}]);function Ct(e){let{children:t}=e;return s.createElement(It,null,t)}function Qt(e){let{error:t,tryAgain:n}=e;return s.createElement("main",{className:"container margin-vert--xl"},s.createElement("div",{className:"row"},s.createElement("div",{className:"col col--6 col--offset-3"},s.createElement("h1",{className:"hero__title"},s.createElement(l.Z,{id:"theme.ErrorPageContent.title",description:"The title of the fallback page when the page crashed"},"This page crashed.")),s.createElement("div",{className:"margin-vert--lg"},s.createElement(st,{onClick:n,className:"button button--primary shadow--lw"})),s.createElement("hr",null),s.createElement("div",{className:"margin-vert--md"},s.createElement(rt,{error:t})))))}const Rt="mainWrapper_z2l0";function Dt(e){const{children:t,noFooter:n,wrapperClassName:i,title:c,description:l}=e;return(0,b.t)(),s.createElement(Ct,null,s.createElement(a.d,{title:c,description:l}),s.createElement(x,null),s.createElement(Q,null),s.createElement(ut,null),s.createElement("div",{id:p,className:(0,r.Z)(f.k.wrapper.main,Rt,i)},s.createElement(o.Z,{fallback:e=>s.createElement(Qt,e)},t)),!n&&s.createElement(wt,null))}},21327:(e,t,n)=>{"use strict";n.d(t,{Z:()=>p});var s=n(83117),r=n(67294),o=n(39960),a=n(44996),i=n(52263),c=n(86668),l=n(50941);function d(e){let{logo:t,alt:n,imageClassName:s}=e;const o={light:(0,a.Z)(t.src),dark:(0,a.Z)(t.srcDark||t.src)},i=r.createElement(l.Z,{className:t.className,sources:o,height:t.height,width:t.width,alt:n,style:t.style});return s?r.createElement("div",{className:s},i):i}function p(e){var t;const{siteConfig:{title:n}}=(0,i.Z)(),{navbar:{title:l,logo:p}}=(0,c.L)(),{imageClassName:u,titleClassName:m,...h}=e,g=(0,a.Z)((null==p?void 0:p.href)||"/"),f=l?"":n,b=null!=(t=null==p?void 0:p.alt)?t:f;return r.createElement(o.Z,(0,s.Z)({to:g},h,(null==p?void 0:p.target)&&{target:p.target}),p&&r.createElement(d,{logo:p,alt:b,imageClassName:u}),null!=l&&r.createElement("b",{className:m},l))}},90197:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=n(67294),r=n(35742);function o(e){let{locale:t,version:n,tag:o}=e;const a=t;return s.createElement(r.Z,null,t&&s.createElement("meta",{name:"docusaurus_locale",content:t}),n&&s.createElement("meta",{name:"docusaurus_version",content:n}),o&&s.createElement("meta",{name:"docusaurus_tag",content:o}),a&&s.createElement("meta",{name:"docsearch:language",content:a}),n&&s.createElement("meta",{name:"docsearch:version",content:n}),o&&s.createElement("meta",{name:"docsearch:docusaurus_tag",content:o}))}},50941:(e,t,n)=>{"use strict";n.d(t,{Z:()=>l});var s=n(83117),r=n(67294),o=n(86010),a=n(72389),i=n(92949);const c={themedImage:"themedImage_ToTc","themedImage--light":"themedImage--light_HNdA","themedImage--dark":"themedImage--dark_i4oU"};function l(e){const t=(0,a.Z)(),{colorMode:n}=(0,i.I)(),{sources:l,className:d,alt:p,...u}=e,m=t?"dark"===n?["dark"]:["light"]:["light","dark"];return r.createElement(r.Fragment,null,m.map((e=>r.createElement("img",(0,s.Z)({key:e,src:l[e],alt:p,className:(0,o.Z)(c.themedImage,c["themedImage--"+e],d)},u)))))}},86043:(e,t,n)=>{"use strict";n.d(t,{u:()=>i,z:()=>g});var s=n(83117),r=n(67294),o=n(10412),a=n(91442);function i(e){let{initialState:t}=e;const[n,s]=(0,r.useState)(null!=t&&t),o=(0,r.useCallback)((()=>{s((e=>!e))}),[]);return{collapsed:n,setCollapsed:s,toggleCollapsed:o}}const c={display:"none",overflow:"hidden",height:"0px"},l={display:"block",overflow:"visible",height:"auto"};function d(e,t){const n=t?c:l;e.style.display=n.display,e.style.overflow=n.overflow,e.style.height=n.height}function p(e){let{collapsibleRef:t,collapsed:n,animation:s}=e;const o=(0,r.useRef)(!1);(0,r.useEffect)((()=>{const e=t.current;function r(){var t,n;const r=e.scrollHeight,o=null!=(t=null==s?void 0:s.duration)?t:function(e){if((0,a.n)())return 1;const t=e/36;return Math.round(10*(4+15*t**.25+t/5))}(r);return{transition:"height "+o+"ms "+(null!=(n=null==s?void 0:s.easing)?n:"ease-in-out"),height:r+"px"}}function i(){const t=r();e.style.transition=t.transition,e.style.height=t.height}if(!o.current)return d(e,n),void(o.current=!0);return e.style.willChange="height",function(){const t=requestAnimationFrame((()=>{n?(i(),requestAnimationFrame((()=>{e.style.height=c.height,e.style.overflow=c.overflow}))):(e.style.display="block",requestAnimationFrame((()=>{i()})))}));return()=>cancelAnimationFrame(t)}()}),[t,n,s])}function u(e){if(!o.Z.canUseDOM)return e?c:l}function m(e){let{as:t="div",collapsed:n,children:s,animation:o,onCollapseTransitionEnd:a,className:i,disableSSRStyle:c}=e;const l=(0,r.useRef)(null);return p({collapsibleRef:l,collapsed:n,animation:o}),r.createElement(t,{ref:l,style:c?void 0:u(n),onTransitionEnd:e=>{"height"===e.propertyName&&(d(l.current,n),null==a||a(n))},className:i},s)}function h(e){let{collapsed:t,...n}=e;const[o,a]=(0,r.useState)(!t),[i,c]=(0,r.useState)(t);return(0,r.useLayoutEffect)((()=>{t||a(!0)}),[t]),(0,r.useLayoutEffect)((()=>{o&&c(t)}),[o,t]),o?r.createElement(m,(0,s.Z)({},n,{collapsed:i})):null}function g(e){let{lazy:t,...n}=e;const s=t?h:m;return r.createElement(s,n)}},59689:(e,t,n)=>{"use strict";n.d(t,{nT:()=>h,pl:()=>m});var s=n(67294),r=n(72389),o=n(50012),a=n(902),i=n(86668);const c=(0,o.WA)("docusaurus.announcement.dismiss"),l=(0,o.WA)("docusaurus.announcement.id"),d=()=>"true"===c.get(),p=e=>c.set(String(e)),u=s.createContext(null);function m(e){let{children:t}=e;const n=function(){const{announcementBar:e}=(0,i.L)(),t=(0,r.Z)(),[n,o]=(0,s.useState)((()=>!!t&&d()));(0,s.useEffect)((()=>{o(d())}),[]);const a=(0,s.useCallback)((()=>{p(!0),o(!0)}),[]);return(0,s.useEffect)((()=>{if(!e)return;const{id:t}=e;let n=l.get();"annoucement-bar"===n&&(n="announcement-bar");const s=t!==n;l.set(t),s&&p(!1),!s&&d()||o(!1)}),[e]),(0,s.useMemo)((()=>({isActive:!!e&&!n,close:a})),[e,n,a])}();return s.createElement(u.Provider,{value:n},t)}function h(){const e=(0,s.useContext)(u);if(!e)throw new a.i6("AnnouncementBarProvider");return e}},92949:(e,t,n)=>{"use strict";n.d(t,{I:()=>f,S:()=>g});var s=n(67294),r=n(10412),o=n(902),a=n(50012),i=n(86668);const c=s.createContext(void 0),l="theme",d=(0,a.WA)(l),p="light",u="dark",m=e=>e===u?u:p;function h(){const{colorMode:{defaultMode:e,disableSwitch:t,respectPrefersColorScheme:n}}=(0,i.L)(),[o,a]=(0,s.useState)((e=>r.Z.canUseDOM?m(document.documentElement.getAttribute("data-theme")):m(e))(e));(0,s.useEffect)((()=>{t&&d.del()}),[t]);const c=(0,s.useCallback)((function(t,s){void 0===s&&(s={});const{persist:r=!0}=s;t?(a(t),r&&(e=>{d.set(m(e))})(t)):(a(n?window.matchMedia("(prefers-color-scheme: dark)").matches?u:p:e),d.del())}),[n,e]);(0,s.useEffect)((()=>{document.documentElement.setAttribute("data-theme",m(o))}),[o]),(0,s.useEffect)((()=>{if(t)return;const e=e=>{if(e.key!==l)return;const t=d.get();null!==t&&c(m(t))};return window.addEventListener("storage",e),()=>window.removeEventListener("storage",e)}),[t,c]);const h=(0,s.useRef)(!1);return(0,s.useEffect)((()=>{if(t&&!n)return;const e=window.matchMedia("(prefers-color-scheme: dark)"),s=()=>{window.matchMedia("print").matches||h.current?h.current=window.matchMedia("print").matches:c(null)};return e.addListener(s),()=>e.removeListener(s)}),[c,t,n]),(0,s.useMemo)((()=>({colorMode:o,setColorMode:c,get isDarkTheme(){return o===u},setLightTheme(){c(p)},setDarkTheme(){c(u)}})),[o,c])}function g(e){let{children:t}=e;const n=h();return s.createElement(c.Provider,{value:n},t)}function f(){const e=(0,s.useContext)(c);if(null==e)throw new o.i6("ColorModeProvider","Please see https://docusaurus.io/docs/api/themes/configuration#use-color-mode.");return e}},60373:(e,t,n)=>{"use strict";n.d(t,{J:()=>x,L5:()=>b,Oh:()=>v});var s=n(67294),r=n(80143),o=n(29935),a=n(86668),i=n(53438),c=n(902),l=n(50012);const d=e=>"docs-preferred-version-"+e,p=(e,t,n)=>{(0,l.WA)(d(e),{persistence:t}).set(n)},u=(e,t)=>(0,l.WA)(d(e),{persistence:t}).get(),m=(e,t)=>{(0,l.WA)(d(e),{persistence:t}).del()};const h=s.createContext(null);function g(){const e=(0,r._r)(),t=(0,a.L)().docs.versionPersistence,n=(0,s.useMemo)((()=>Object.keys(e)),[e]),[o,i]=(0,s.useState)((()=>(e=>Object.fromEntries(e.map((e=>[e,{preferredVersionName:null}]))))(n)));(0,s.useEffect)((()=>{i(function(e){let{pluginIds:t,versionPersistence:n,allDocsData:s}=e;function r(e){const t=u(e,n);return s[e].versions.some((e=>e.name===t))?{preferredVersionName:t}:(m(e,n),{preferredVersionName:null})}return Object.fromEntries(t.map((e=>[e,r(e)])))}({allDocsData:e,versionPersistence:t,pluginIds:n}))}),[e,t,n]);return[o,(0,s.useMemo)((()=>({savePreferredVersion:function(e,n){p(e,t,n),i((t=>({...t,[e]:{preferredVersionName:n}})))}})),[t])]}function f(e){let{children:t}=e;const n=g();return s.createElement(h.Provider,{value:n},t)}function b(e){let{children:t}=e;return i.cE?s.createElement(f,null,t):s.createElement(s.Fragment,null,t)}function y(){const e=(0,s.useContext)(h);if(!e)throw new c.i6("DocsPreferredVersionContextProvider");return e}function x(e){var t;void 0===e&&(e=o.m);const n=(0,r.zh)(e),[a,i]=y(),{preferredVersionName:c}=a[e];return{preferredVersion:null!=(t=n.versions.find((e=>e.name===c)))?t:null,savePreferredVersionName:(0,s.useCallback)((t=>{i.savePreferredVersion(e,t)}),[i,e])}}function v(){const e=(0,r._r)(),[t]=y();function n(n){var s;const r=e[n],{preferredVersionName:o}=t[n];return null!=(s=r.versions.find((e=>e.name===o)))?s:null}const s=Object.keys(e);return Object.fromEntries(s.map((e=>[e,n(e)])))}},1116:(e,t,n)=>{"use strict";n.d(t,{V:()=>c,b:()=>i});var s=n(67294),r=n(902);const o=Symbol("EmptyContext"),a=s.createContext(o);function i(e){let{children:t,name:n,items:r}=e;const o=(0,s.useMemo)((()=>n&&r?{name:n,items:r}:null),[n,r]);return s.createElement(a.Provider,{value:o},t)}function c(){const e=(0,s.useContext)(a);if(e===o)throw new r.i6("DocsSidebarProvider");return e}},93163:(e,t,n)=>{"use strict";n.d(t,{M:()=>p,e:()=>u});var s=n(67294),r=n(13102),o=n(87524),a=n(91980),i=n(86668),c=n(902);const l=s.createContext(void 0);function d(){const e=function(){const e=(0,r.HY)(),{items:t}=(0,i.L)().navbar;return 0===t.length&&!e.component}(),t=(0,o.i)(),n=!e&&"mobile"===t,[c,l]=(0,s.useState)(!1);(0,a.Rb)((()=>{if(c)return l(!1),!1}));const d=(0,s.useCallback)((()=>{l((e=>!e))}),[]);return(0,s.useEffect)((()=>{"desktop"===t&&l(!1)}),[t]),(0,s.useMemo)((()=>({disabled:e,shouldRender:n,toggle:d,shown:c})),[e,n,d,c])}function p(e){let{children:t}=e;const n=d();return s.createElement(l.Provider,{value:n},t)}function u(){const e=s.useContext(l);if(void 0===e)throw new c.i6("NavbarMobileSidebarProvider");return e}},13102:(e,t,n)=>{"use strict";n.d(t,{HY:()=>i,Zo:()=>c,n2:()=>a});var s=n(67294),r=n(902);const o=s.createContext(null);function a(e){let{children:t}=e;const n=(0,s.useState)({component:null,props:null});return s.createElement(o.Provider,{value:n},t)}function i(){const e=(0,s.useContext)(o);if(!e)throw new r.i6("NavbarSecondaryMenuContentProvider");return e[0]}function c(e){let{component:t,props:n}=e;const a=(0,s.useContext)(o);if(!a)throw new r.i6("NavbarSecondaryMenuContentProvider");const[,i]=a,c=(0,r.Ql)(n);return(0,s.useEffect)((()=>{i({component:t,props:c})}),[i,t,c]),(0,s.useEffect)((()=>()=>i({component:null,props:null})),[i]),null}},19727:(e,t,n)=>{"use strict";n.d(t,{h:()=>r,t:()=>o});var s=n(67294);const r="navigation-with-keyboard";function o(){(0,s.useEffect)((()=>{function e(e){"keydown"===e.type&&"Tab"===e.key&&document.body.classList.add(r),"mousedown"===e.type&&document.body.classList.remove(r)}return document.addEventListener("keydown",e),document.addEventListener("mousedown",e),()=>{document.body.classList.remove(r),document.removeEventListener("keydown",e),document.removeEventListener("mousedown",e)}}),[])}},66177:(e,t,n)=>{"use strict";n.d(t,{K:()=>a,M:()=>i});var s=n(67294),r=n(52263),o=n(91980);function a(){return(0,o.Nc)("q")}function i(){const{siteConfig:{baseUrl:e,themeConfig:t}}=(0,r.Z)(),{algolia:{searchPagePath:n}}=t;return(0,s.useCallback)((t=>""+e+n+"?q="+encodeURIComponent(t)),[e,n])}},87524:(e,t,n)=>{"use strict";n.d(t,{i:()=>l});var s=n(67294),r=n(10412);const o="desktop",a="mobile",i="ssr";function c(){return r.Z.canUseDOM?window.innerWidth>996?o:a:i}function l(){const[e,t]=(0,s.useState)((()=>c()));return(0,s.useEffect)((()=>{function e(){t(c())}return window.addEventListener("resize",e),()=>{window.removeEventListener("resize",e),clearTimeout(undefined)}}),[]),e}},35281:(e,t,n)=>{"use strict";n.d(t,{k:()=>s});const s={page:{blogListPage:"blog-list-page",blogPostPage:"blog-post-page",blogTagsListPage:"blog-tags-list-page",blogTagPostListPage:"blog-tags-post-list-page",docsDocPage:"docs-doc-page",docsTagsListPage:"docs-tags-list-page",docsTagDocListPage:"docs-tags-doc-list-page",mdxPage:"mdx-page"},wrapper:{main:"main-wrapper",blogPages:"blog-wrapper",docsPages:"docs-wrapper",mdxPages:"mdx-wrapper"},common:{editThisPage:"theme-edit-this-page",lastUpdated:"theme-last-updated",backToTopButton:"theme-back-to-top-button",codeBlock:"theme-code-block",admonition:"theme-admonition",admonitionType:e=>"theme-admonition-"+e},layout:{},docs:{docVersionBanner:"theme-doc-version-banner",docVersionBadge:"theme-doc-version-badge",docBreadcrumbs:"theme-doc-breadcrumbs",docMarkdown:"theme-doc-markdown",docTocMobile:"theme-doc-toc-mobile",docTocDesktop:"theme-doc-toc-desktop",docFooter:"theme-doc-footer",docFooterTagsRow:"theme-doc-footer-tags-row",docFooterEditMetaRow:"theme-doc-footer-edit-meta-row",docSidebarContainer:"theme-doc-sidebar-container",docSidebarMenu:"theme-doc-sidebar-menu",docSidebarItemCategory:"theme-doc-sidebar-item-category",docSidebarItemLink:"theme-doc-sidebar-item-link",docSidebarItemCategoryLevel:e=>"theme-doc-sidebar-item-category-level-"+e,docSidebarItemLinkLevel:e=>"theme-doc-sidebar-item-link-level-"+e},blog:{}}},91442:(e,t,n)=>{"use strict";function s(){return window.matchMedia("(prefers-reduced-motion: reduce)").matches}n.d(t,{n:()=>s})},53438:(e,t,n)=>{"use strict";n.d(t,{Wl:()=>u,_F:()=>h,cE:()=>p,hI:()=>v,lO:()=>b,oz:()=>y,s1:()=>f,vY:()=>x});var s=n(67294),r=n(16550),o=n(18790),a=n(80143),i=n(60373),c=n(1116),l=n(67392),d=n(48596);const p=!!a._r;function u(e){if(e.href)return e.href;for(const t of e.items){if("link"===t.type)return t.href;if("category"===t.type){const e=u(t);if(e)return e}}}const m=(e,t)=>void 0!==e&&(0,d.Mg)(e,t);function h(e,t){return"link"===e.type?m(e.href,t):"category"===e.type&&(m(e.href,t)||((e,t)=>e.some((e=>h(e,t))))(e.items,t))}function g(e){let{sidebarItems:t,pathname:n,onlyCategories:s=!1}=e;const r=[];return function e(t){for(const o of t)if("category"===o.type&&((0,d.Mg)(o.href,n)||e(o.items))||"link"===o.type&&(0,d.Mg)(o.href,n)){return s&&"category"!==o.type||r.unshift(o),!0}return!1}(t),r}function f(){var e;const t=(0,c.V)(),{pathname:n}=(0,r.TH)();return!1!==(null==(e=(0,a.gA)())?void 0:e.pluginData.breadcrumbs)&&t?g({sidebarItems:t.items,pathname:n}):null}function b(e){const{activeVersion:t}=(0,a.Iw)(e),{preferredVersion:n}=(0,i.J)(e),r=(0,a.yW)(e);return(0,s.useMemo)((()=>(0,l.j)([t,n,r].filter(Boolean))),[t,n,r])}function y(e,t){const n=b(t);return(0,s.useMemo)((()=>{const t=n.flatMap((e=>e.sidebars?Object.entries(e.sidebars):[])),s=t.find((t=>t[0]===e));if(!s)throw new Error("Can't find any sidebar with id \""+e+'" in version'+(n.length>1?"s":"")+" "+n.map((e=>e.name)).join(", ")+'".\nAvailable sidebar ids are:\n- '+t.map((e=>e[0])).join("\n- "));return s[1]}),[e,n])}function x(e,t){const n=b(t);return(0,s.useMemo)((()=>{const t=n.flatMap((e=>e.docs)),s=t.find((t=>t.id===e));if(!s){if(n.flatMap((e=>e.draftIds)).includes(e))return null;throw new Error("Couldn't find any doc with id \""+e+'" in version'+(n.length>1?"s":"")+' "'+n.map((e=>e.name)).join(", ")+'".\nAvailable doc ids are:\n- '+(0,l.j)(t.map((e=>e.id))).join("\n- "))}return s}),[e,n])}function v(e){let{route:t,versionMetadata:n}=e;const s=(0,r.TH)(),a=t.routes,i=a.find((e=>(0,r.LX)(s.pathname,e)));if(!i)return null;const c=i.sidebar,l=c?n.docsSidebars[c]:void 0;return{docElement:(0,o.H)(a),sidebarName:c,sidebarItems:l}}},82128:(e,t,n)=>{"use strict";n.d(t,{p:()=>r});var s=n(52263);function r(e){const{siteConfig:t}=(0,s.Z)(),{title:n,titleDelimiter:r}=t;return null!=e&&e.trim().length?e.trim()+" "+r+" "+n:n}},91980:(e,t,n)=>{"use strict";n.d(t,{Nc:()=>l,Rb:()=>i,_X:()=>c});var s=n(67294),r=n(16550),o=n(61688),a=n(902);function i(e){!function(e){const t=(0,r.k6)(),n=(0,a.zX)(e);(0,s.useEffect)((()=>t.block(((e,t)=>n(e,t)))),[t,n])}(((t,n)=>{if("POP"===n)return e(t,n)}))}function c(e){return function(e){const t=(0,r.k6)();return(0,o.useSyncExternalStore)(t.listen,(()=>e(t)),(()=>e(t)))}((t=>null===e?null:new URLSearchParams(t.location.search).get(e)))}function l(e){var t;const n=null!=(t=c(e))?t:"",o=function(){const e=(0,r.k6)();return(0,s.useCallback)(((t,n,s)=>{const r=new URLSearchParams(e.location.search);n?r.set(t,n):r.delete(t),(null!=s&&s.push?e.push:e.replace)({search:r.toString()})}),[e])}();return[n,(0,s.useCallback)(((t,n)=>{o(e,t,n)}),[o,e])]}},67392:(e,t,n)=>{"use strict";function s(e,t){return void 0===t&&(t=(e,t)=>e===t),e.filter(((n,s)=>e.findIndex((e=>t(e,n)))!==s))}function r(e){return Array.from(new Set(e))}n.d(t,{j:()=>r,l:()=>s})},10833:(e,t,n)=>{"use strict";n.d(t,{FG:()=>u,d:()=>d,VC:()=>m});var s=n(67294),r=n(86010),o=n(35742),a=n(30226);function i(){const e=s.useContext(a._);if(!e)throw new Error("Unexpected: no Docusaurus route context found");return e}var c=n(44996),l=n(82128);function d(e){let{title:t,description:n,keywords:r,image:a,children:i}=e;const d=(0,l.p)(t),{withBaseUrl:p}=(0,c.C)(),u=a?p(a,{absolute:!0}):void 0;return s.createElement(o.Z,null,t&&s.createElement("title",null,d),t&&s.createElement("meta",{property:"og:title",content:d}),n&&s.createElement("meta",{name:"description",content:n}),n&&s.createElement("meta",{property:"og:description",content:n}),r&&s.createElement("meta",{name:"keywords",content:Array.isArray(r)?r.join(","):r}),u&&s.createElement("meta",{property:"og:image",content:u}),u&&s.createElement("meta",{name:"twitter:image",content:u}),i)}const p=s.createContext(void 0);function u(e){let{className:t,children:n}=e;const a=s.useContext(p),i=(0,r.Z)(a,t);return s.createElement(p.Provider,{value:i},s.createElement(o.Z,null,s.createElement("html",{className:i})),n)}function m(e){let{children:t}=e;const n=i(),o="plugin-"+n.plugin.name.replace(/docusaurus-(?:plugin|theme)-(?:content-)?/gi,"");const a="plugin-id-"+n.plugin.id;return s.createElement(u,{className:(0,r.Z)(o,a)},t)}},902:(e,t,n)=>{"use strict";n.d(t,{D9:()=>a,Qc:()=>l,Ql:()=>c,i6:()=>i,zX:()=>o});var s=n(67294);const r=n(10412).Z.canUseDOM?s.useLayoutEffect:s.useEffect;function o(e){const t=(0,s.useRef)(e);return r((()=>{t.current=e}),[e]),(0,s.useCallback)((function(){return t.current(...arguments)}),[])}function a(e){const t=(0,s.useRef)();return r((()=>{t.current=e})),t.current}class i extends Error{constructor(e,t){var n,s,r,o;super(),this.name="ReactContextError",this.message="Hook "+(null!=(n=null==(s=this.stack)||null==(r=s.split("\n")[1])||null==(o=r.match(/at (?:\w+\.)?(?<name>\w+)/))?void 0:o.groups.name)?n:"")+" is called outside the <"+e+">. "+(null!=t?t:"")}}function c(e){const t=Object.entries(e);return t.sort(((e,t)=>e[0].localeCompare(t[0]))),(0,s.useMemo)((()=>e),t.flat())}function l(e){return t=>{let{children:n}=t;return s.createElement(s.Fragment,null,e.reduceRight(((e,t)=>s.createElement(t,null,e)),n))}}},98022:(e,t,n)=>{"use strict";function s(e,t){return void 0!==e&&void 0!==t&&new RegExp(e,"gi").test(t)}n.d(t,{F:()=>s})},48596:(e,t,n)=>{"use strict";n.d(t,{Mg:()=>a,Ns:()=>i});var s=n(67294),r=n(723),o=n(52263);function a(e,t){const n=e=>{var t;return null==(t=!e||e.endsWith("/")?e:e+"/")?void 0:t.toLowerCase()};return n(e)===n(t)}function i(){const{baseUrl:e}=(0,o.Z)().siteConfig;return(0,s.useMemo)((()=>function(e){let{baseUrl:t,routes:n}=e;function s(e){return e.path===t&&!0===e.exact}function r(e){return e.path===t&&!e.exact}return function e(t){if(0===t.length)return;return t.find(s)||e(t.filter(r).flatMap((e=>{var t;return null!=(t=e.routes)?t:[]})))}(n)}({routes:r.Z,baseUrl:e})),[e])}},12466:(e,t,n)=>{"use strict";n.d(t,{Ct:()=>m,OC:()=>c,RF:()=>p,o5:()=>u});var s=n(67294),r=n(10412),o=n(72389),a=n(902);const i=s.createContext(void 0);function c(e){let{children:t}=e;const n=function(){const e=(0,s.useRef)(!0);return(0,s.useMemo)((()=>({scrollEventsEnabledRef:e,enableScrollEvents:()=>{e.current=!0},disableScrollEvents:()=>{e.current=!1}})),[])}();return s.createElement(i.Provider,{value:n},t)}function l(){const e=(0,s.useContext)(i);if(null==e)throw new a.i6("ScrollControllerProvider");return e}const d=()=>r.Z.canUseDOM?{scrollX:window.pageXOffset,scrollY:window.pageYOffset}:null;function p(e,t){void 0===t&&(t=[]);const{scrollEventsEnabledRef:n}=l(),r=(0,s.useRef)(d()),o=(0,a.zX)(e);(0,s.useEffect)((()=>{const e=()=>{if(!n.current)return;const e=d();o(e,r.current),r.current=e},t={passive:!0};return e(),window.addEventListener("scroll",e,t),()=>window.removeEventListener("scroll",e,t)}),[o,n,...t])}function u(){const e=l(),t=function(){const e=(0,s.useRef)({elem:null,top:0}),t=(0,s.useCallback)((t=>{e.current={elem:t,top:t.getBoundingClientRect().top}}),[]),n=(0,s.useCallback)((()=>{const{current:{elem:t,top:n}}=e;if(!t)return{restored:!1};const s=t.getBoundingClientRect().top-n;return s&&window.scrollBy({left:0,top:s}),e.current={elem:null,top:0},{restored:0!==s}}),[]);return(0,s.useMemo)((()=>({save:t,restore:n})),[n,t])}(),n=(0,s.useRef)(void 0),r=(0,s.useCallback)((s=>{t.save(s),e.disableScrollEvents(),n.current=()=>{const{restored:s}=t.restore();if(n.current=void 0,s){const t=()=>{e.enableScrollEvents(),window.removeEventListener("scroll",t)};window.addEventListener("scroll",t)}else e.enableScrollEvents()}}),[e,t]);return(0,s.useLayoutEffect)((()=>{queueMicrotask((()=>null==n.current?void 0:n.current()))})),{blockElementScrollPositionUntilNextRender:r}}function m(){const e=(0,s.useRef)(null),t=(0,o.Z)()&&"smooth"===getComputedStyle(document.documentElement).scrollBehavior;return{startScroll:n=>{e.current=t?function(e){return window.scrollTo({top:e,behavior:"smooth"}),()=>{}}(n):function(e){let t=null;const n=document.documentElement.scrollTop>e;return function s(){const r=document.documentElement.scrollTop;(n&&r>e||!n&&r<e)&&(t=requestAnimationFrame(s),window.scrollTo(0,Math.floor(.85*(r-e))+e))}(),()=>t&&cancelAnimationFrame(t)}(n)},cancelScroll:()=>null==e.current?void 0:e.current()}}},43320:(e,t,n)=>{"use strict";n.d(t,{HX:()=>a,_q:()=>c,os:()=>i});var s=n(80143),r=n(52263),o=n(60373);const a="default";function i(e,t){return"docs-"+e+"-"+t}function c(){const{i18n:e}=(0,r.Z)(),t=(0,s._r)(),n=(0,s.WS)(),c=(0,o.Oh)();const l=[a,...Object.keys(t).map((function(e){var s;const r=(null==n?void 0:n.activePlugin.pluginId)===e?n.activeVersion:void 0,o=c[e],a=t[e].versions.find((e=>e.isLast));return i(e,(null!=(s=null!=r?r:o)?s:a).name)}))];return{locale:e.currentLocale,tags:l}}},50012:(e,t,n)=>{"use strict";n.d(t,{Nk:()=>p,WA:()=>d});var s=n(67294),r=n(61688);const o="localStorage";function a(e){let{key:t,oldValue:n,newValue:s,storage:r}=e;if(n===s)return;const o=document.createEvent("StorageEvent");o.initStorageEvent("storage",!1,!1,t,n,s,window.location.href,r),window.dispatchEvent(o)}function i(e){if(void 0===e&&(e=o),"undefined"==typeof window)throw new Error("Browser storage is not available on Node.js/Docusaurus SSR process.");if("none"===e)return null;try{return window[e]}catch(n){return t=n,c||(console.warn("Docusaurus browser storage is not available.\nPossible reasons: running Docusaurus in an iframe, in an incognito browser session, or using too strict browser privacy settings.",t),c=!0),null}var t}let c=!1;const l={get:()=>null,set:()=>{},del:()=>{},listen:()=>()=>{}};function d(e,t){if("undefined"==typeof window)return function(e){function t(){throw new Error('Illegal storage API usage for storage key "'+e+'".\nDocusaurus storage APIs are not supposed to be called on the server-rendering process.\nPlease only call storage APIs in effects and event handlers.')}return{get:t,set:t,del:t,listen:t}}(e);const n=i(null==t?void 0:t.persistence);return null===n?l:{get:()=>{try{return n.getItem(e)}catch(t){return console.error("Docusaurus storage error, can't get key="+e,t),null}},set:t=>{try{const s=n.getItem(e);n.setItem(e,t),a({key:e,oldValue:s,newValue:t,storage:n})}catch(s){console.error("Docusaurus storage error, can't set "+e+"="+t,s)}},del:()=>{try{const t=n.getItem(e);n.removeItem(e),a({key:e,oldValue:t,newValue:null,storage:n})}catch(t){console.error("Docusaurus storage error, can't delete key="+e,t)}},listen:t=>{try{const s=s=>{s.storageArea===n&&s.key===e&&t(s)};return window.addEventListener("storage",s),()=>window.removeEventListener("storage",s)}catch(s){return console.error("Docusaurus storage error, can't listen for changes of key="+e,s),()=>{}}}}}function p(e,t){const n=(0,s.useRef)((()=>null===e?l:d(e,t))).current(),o=(0,s.useCallback)((e=>"undefined"==typeof window?()=>{}:n.listen(e)),[n]);return[(0,r.useSyncExternalStore)(o,(()=>"undefined"==typeof window?null:n.get()),(()=>null)),n]}},94711:(e,t,n)=>{"use strict";n.d(t,{l:()=>o});var s=n(52263),r=n(16550);function o(){const{siteConfig:{baseUrl:e,url:t},i18n:{defaultLocale:n,currentLocale:o}}=(0,s.Z)(),{pathname:a}=(0,r.TH)(),i=o===n?e:e.replace("/"+o+"/","/"),c=a.replace(e,"");return{createUrl:function(e){let{locale:s,fullyQualified:r}=e;return""+(r?t:"")+function(e){return e===n?""+i:""+i+e+"/"}(s)+c}}}},85936:(e,t,n)=>{"use strict";n.d(t,{S:()=>a});var s=n(67294),r=n(16550),o=n(902);function a(e){const t=(0,r.TH)(),n=(0,o.D9)(t),a=(0,o.zX)(e);(0,s.useEffect)((()=>{n&&t!==n&&a({location:t,previousLocation:n})}),[a,t,n])}},86668:(e,t,n)=>{"use strict";n.d(t,{L:()=>r});var s=n(52263);function r(){return(0,s.Z)().siteConfig.themeConfig}},6278:(e,t,n)=>{"use strict";n.d(t,{L:()=>r});var s=n(52263);function r(){const{siteConfig:{themeConfig:e}}=(0,s.Z)();return e}},239:(e,t,n)=>{"use strict";n.d(t,{l:()=>i});var s=n(67294),r=n(98022),o=n(44996),a=n(6278);function i(){const{withBaseUrl:e}=(0,o.C)(),{algolia:{externalUrlRegex:t,replaceSearchResultPathname:n}}=(0,a.L)();return(0,s.useCallback)((s=>{const o=new URL(s);if((0,r.F)(t,o.href))return s;const a=""+(o.pathname+o.hash);return e(function(e,t){return t?e.replaceAll(new RegExp(t.from,"g"),t.to):e}(a,n))}),[e,t,n])}},8802:(e,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.default=function(e,t){const{trailingSlash:n,baseUrl:s}=t;if(e.startsWith("#"))return e;if(void 0===n)return e;const[r]=e.split(/[#?]/),o="/"===r||r===s?r:(a=r,n?function(e){return e.endsWith("/")?e:e+"/"}(a):function(e){return e.endsWith("/")?e.slice(0,-1):e}(a));var a;return e.replace(r,o)}},54143:(e,t)=>{"use strict";Object.defineProperty(t,"__esModule",{value:!0}),t.getErrorCausalChain=void 0,t.getErrorCausalChain=function e(t){return t.cause?[t,...e(t.cause)]:[t]}},18780:function(e,t,n){"use strict";var s=this&&this.__importDefault||function(e){return e&&e.__esModule?e:{default:e}};Object.defineProperty(t,"__esModule",{value:!0}),t.getErrorCausalChain=t.applyTrailingSlash=t.blogPostContainerID=void 0,t.blogPostContainerID="__blog-post-container";var r=n(8802);Object.defineProperty(t,"applyTrailingSlash",{enumerable:!0,get:function(){return s(r).default}});var o=n(54143);Object.defineProperty(t,"getErrorCausalChain",{enumerable:!0,get:function(){return o.getErrorCausalChain}})},86010:(e,t,n)=>{"use strict";function s(e){var t,n,r="";if("string"==typeof e||"number"==typeof e)r+=e;else if("object"==typeof e)if(Array.isArray(e))for(t=0;t<e.length;t++)e[t]&&(n=s(e[t]))&&(r&&(r+=" "),r+=n);else for(t in e)e[t]&&(r&&(r+=" "),r+=t);return r}n.d(t,{Z:()=>r});const r=function(){for(var e,t,n=0,r="";n<arguments.length;)(e=arguments[n++])&&(t=s(e))&&(r&&(r+=" "),r+=t);return r}},99318:(e,t,n)=>{"use strict";n.d(t,{lX:()=>v,q_:()=>k,ob:()=>m,PP:()=>w,Ep:()=>u});var s=n(83117);function r(e){return"/"===e.charAt(0)}function o(e,t){for(var n=t,s=n+1,r=e.length;s<r;n+=1,s+=1)e[n]=e[s];e.pop()}const a=function(e,t){void 0===t&&(t="");var n,s=e&&e.split("/")||[],a=t&&t.split("/")||[],i=e&&r(e),c=t&&r(t),l=i||c;if(e&&r(e)?a=s:s.length&&(a.pop(),a=a.concat(s)),!a.length)return"/";if(a.length){var d=a[a.length-1];n="."===d||".."===d||""===d}else n=!1;for(var p=0,u=a.length;u>=0;u--){var m=a[u];"."===m?o(a,u):".."===m?(o(a,u),p++):p&&(o(a,u),p--)}if(!l)for(;p--;p)a.unshift("..");!l||""===a[0]||a[0]&&r(a[0])||a.unshift("");var h=a.join("/");return n&&"/"!==h.substr(-1)&&(h+="/"),h};var i=n(2177);function c(e){return"/"===e.charAt(0)?e:"/"+e}function l(e){return"/"===e.charAt(0)?e.substr(1):e}function d(e,t){return function(e,t){return 0===e.toLowerCase().indexOf(t.toLowerCase())&&-1!=="/?#".indexOf(e.charAt(t.length))}(e,t)?e.substr(t.length):e}function p(e){return"/"===e.charAt(e.length-1)?e.slice(0,-1):e}function u(e){var t=e.pathname,n=e.search,s=e.hash,r=t||"/";return n&&"?"!==n&&(r+="?"===n.charAt(0)?n:"?"+n),s&&"#"!==s&&(r+="#"===s.charAt(0)?s:"#"+s),r}function m(e,t,n,r){var o;"string"==typeof e?(o=function(e){var t=e||"/",n="",s="",r=t.indexOf("#");-1!==r&&(s=t.substr(r),t=t.substr(0,r));var o=t.indexOf("?");return-1!==o&&(n=t.substr(o),t=t.substr(0,o)),{pathname:t,search:"?"===n?"":n,hash:"#"===s?"":s}}(e),o.state=t):(void 0===(o=(0,s.Z)({},e)).pathname&&(o.pathname=""),o.search?"?"!==o.search.charAt(0)&&(o.search="?"+o.search):o.search="",o.hash?"#"!==o.hash.charAt(0)&&(o.hash="#"+o.hash):o.hash="",void 0!==t&&void 0===o.state&&(o.state=t));try{o.pathname=decodeURI(o.pathname)}catch(i){throw i instanceof URIError?new URIError('Pathname "'+o.pathname+'" could not be decoded. This is likely caused by an invalid percent-encoding.'):i}return n&&(o.key=n),r?o.pathname?"/"!==o.pathname.charAt(0)&&(o.pathname=a(o.pathname,r.pathname)):o.pathname=r.pathname:o.pathname||(o.pathname="/"),o}function h(){var e=null;var t=[];return{setPrompt:function(t){return e=t,function(){e===t&&(e=null)}},confirmTransitionTo:function(t,n,s,r){if(null!=e){var o="function"==typeof e?e(t,n):e;"string"==typeof o?"function"==typeof s?s(o,r):r(!0):r(!1!==o)}else r(!0)},appendListener:function(e){var n=!0;function s(){n&&e.apply(void 0,arguments)}return t.push(s),function(){n=!1,t=t.filter((function(e){return e!==s}))}},notifyListeners:function(){for(var e=arguments.length,n=new Array(e),s=0;s<e;s++)n[s]=arguments[s];t.forEach((function(e){return e.apply(void 0,n)}))}}}var g=!("undefined"==typeof window||!window.document||!window.document.createElement);function f(e,t){t(window.confirm(e))}var b="popstate",y="hashchange";function x(){try{return window.history.state||{}}catch(e){return{}}}function v(e){void 0===e&&(e={}),g||(0,i.Z)(!1);var t,n=window.history,r=(-1===(t=window.navigator.userAgent).indexOf("Android 2.")&&-1===t.indexOf("Android 4.0")||-1===t.indexOf("Mobile Safari")||-1!==t.indexOf("Chrome")||-1!==t.indexOf("Windows Phone"))&&window.history&&"pushState"in window.history,o=!(-1===window.navigator.userAgent.indexOf("Trident")),a=e,l=a.forceRefresh,v=void 0!==l&&l,A=a.getUserConfirmation,S=void 0===A?f:A,E=a.keyLength,L=void 0===E?6:E,M=e.basename?p(c(e.basename)):"";function k(e){var t=e||{},n=t.key,s=t.state,r=window.location,o=r.pathname+r.search+r.hash;return M&&(o=d(o,M)),m(o,s,n)}function _(){return Math.random().toString(36).substr(2,L)}var w=h();function I(e){(0,s.Z)(G,e),G.length=n.length,w.notifyListeners(G.location,G.action)}function C(e){(function(e){return void 0===e.state&&-1===navigator.userAgent.indexOf("CriOS")})(e)||D(k(e.state))}function Q(){D(k(x()))}var R=!1;function D(e){if(R)R=!1,I();else{w.confirmTransitionTo(e,"POP",S,(function(t){t?I({action:"POP",location:e}):function(e){var t=G.location,n=O.indexOf(t.key);-1===n&&(n=0);var s=O.indexOf(e.key);-1===s&&(s=0);var r=n-s;r&&(R=!0,F(r))}(e)}))}}var T=k(x()),O=[T.key];function P(e){return M+u(e)}function F(e){n.go(e)}var N=0;function B(e){1===(N+=e)&&1===e?(window.addEventListener(b,C),o&&window.addEventListener(y,Q)):0===N&&(window.removeEventListener(b,C),o&&window.removeEventListener(y,Q))}var V=!1;var G={length:n.length,action:"POP",location:T,createHref:P,push:function(e,t){var s="PUSH",o=m(e,t,_(),G.location);w.confirmTransitionTo(o,s,S,(function(e){if(e){var t=P(o),a=o.key,i=o.state;if(r)if(n.pushState({key:a,state:i},null,t),v)window.location.href=t;else{var c=O.indexOf(G.location.key),l=O.slice(0,c+1);l.push(o.key),O=l,I({action:s,location:o})}else window.location.href=t}}))},replace:function(e,t){var s="REPLACE",o=m(e,t,_(),G.location);w.confirmTransitionTo(o,s,S,(function(e){if(e){var t=P(o),a=o.key,i=o.state;if(r)if(n.replaceState({key:a,state:i},null,t),v)window.location.replace(t);else{var c=O.indexOf(G.location.key);-1!==c&&(O[c]=o.key),I({action:s,location:o})}else window.location.replace(t)}}))},go:F,goBack:function(){F(-1)},goForward:function(){F(1)},block:function(e){void 0===e&&(e=!1);var t=w.setPrompt(e);return V||(B(1),V=!0),function(){return V&&(V=!1,B(-1)),t()}},listen:function(e){var t=w.appendListener(e);return B(1),function(){B(-1),t()}}};return G}var A="hashchange",S={hashbang:{encodePath:function(e){return"!"===e.charAt(0)?e:"!/"+l(e)},decodePath:function(e){return"!"===e.charAt(0)?e.substr(1):e}},noslash:{encodePath:l,decodePath:c},slash:{encodePath:c,decodePath:c}};function E(e){var t=e.indexOf("#");return-1===t?e:e.slice(0,t)}function L(){var e=window.location.href,t=e.indexOf("#");return-1===t?"":e.substring(t+1)}function M(e){window.location.replace(E(window.location.href)+"#"+e)}function k(e){void 0===e&&(e={}),g||(0,i.Z)(!1);var t=window.history,n=(window.navigator.userAgent.indexOf("Firefox"),e),r=n.getUserConfirmation,o=void 0===r?f:r,a=n.hashType,l=void 0===a?"slash":a,b=e.basename?p(c(e.basename)):"",y=S[l],x=y.encodePath,v=y.decodePath;function k(){var e=v(L());return b&&(e=d(e,b)),m(e)}var _=h();function w(e){(0,s.Z)(V,e),V.length=t.length,_.notifyListeners(V.location,V.action)}var I=!1,C=null;function Q(){var e,t,n=L(),s=x(n);if(n!==s)M(s);else{var r=k(),a=V.location;if(!I&&(t=r,(e=a).pathname===t.pathname&&e.search===t.search&&e.hash===t.hash))return;if(C===u(r))return;C=null,function(e){if(I)I=!1,w();else{var t="POP";_.confirmTransitionTo(e,t,o,(function(n){n?w({action:t,location:e}):function(e){var t=V.location,n=O.lastIndexOf(u(t));-1===n&&(n=0);var s=O.lastIndexOf(u(e));-1===s&&(s=0);var r=n-s;r&&(I=!0,P(r))}(e)}))}}(r)}}var R=L(),D=x(R);R!==D&&M(D);var T=k(),O=[u(T)];function P(e){t.go(e)}var F=0;function N(e){1===(F+=e)&&1===e?window.addEventListener(A,Q):0===F&&window.removeEventListener(A,Q)}var B=!1;var V={length:t.length,action:"POP",location:T,createHref:function(e){var t=document.querySelector("base"),n="";return t&&t.getAttribute("href")&&(n=E(window.location.href)),n+"#"+x(b+u(e))},push:function(e,t){var n="PUSH",s=m(e,void 0,void 0,V.location);_.confirmTransitionTo(s,n,o,(function(e){if(e){var t=u(s),r=x(b+t);if(L()!==r){C=t,function(e){window.location.hash=e}(r);var o=O.lastIndexOf(u(V.location)),a=O.slice(0,o+1);a.push(t),O=a,w({action:n,location:s})}else w()}}))},replace:function(e,t){var n="REPLACE",s=m(e,void 0,void 0,V.location);_.confirmTransitionTo(s,n,o,(function(e){if(e){var t=u(s),r=x(b+t);L()!==r&&(C=t,M(r));var o=O.indexOf(u(V.location));-1!==o&&(O[o]=t),w({action:n,location:s})}}))},go:P,goBack:function(){P(-1)},goForward:function(){P(1)},block:function(e){void 0===e&&(e=!1);var t=_.setPrompt(e);return B||(N(1),B=!0),function(){return B&&(B=!1,N(-1)),t()}},listen:function(e){var t=_.appendListener(e);return N(1),function(){N(-1),t()}}};return V}function _(e,t,n){return Math.min(Math.max(e,t),n)}function w(e){void 0===e&&(e={});var t=e,n=t.getUserConfirmation,r=t.initialEntries,o=void 0===r?["/"]:r,a=t.initialIndex,i=void 0===a?0:a,c=t.keyLength,l=void 0===c?6:c,d=h();function p(e){(0,s.Z)(v,e),v.length=v.entries.length,d.notifyListeners(v.location,v.action)}function g(){return Math.random().toString(36).substr(2,l)}var f=_(i,0,o.length-1),b=o.map((function(e){return m(e,void 0,"string"==typeof e?g():e.key||g())})),y=u;function x(e){var t=_(v.index+e,0,v.entries.length-1),s=v.entries[t];d.confirmTransitionTo(s,"POP",n,(function(e){e?p({action:"POP",location:s,index:t}):p()}))}var v={length:b.length,action:"POP",location:b[f],index:f,entries:b,createHref:y,push:function(e,t){var s="PUSH",r=m(e,t,g(),v.location);d.confirmTransitionTo(r,s,n,(function(e){if(e){var t=v.index+1,n=v.entries.slice(0);n.length>t?n.splice(t,n.length-t,r):n.push(r),p({action:s,location:r,index:t,entries:n})}}))},replace:function(e,t){var s="REPLACE",r=m(e,t,g(),v.location);d.confirmTransitionTo(r,s,n,(function(e){e&&(v.entries[v.index]=r,p({action:s,location:r}))}))},go:x,goBack:function(){x(-1)},goForward:function(){x(1)},canGo:function(e){var t=v.index+e;return t>=0&&t<v.entries.length},block:function(e){return void 0===e&&(e=!1),d.setPrompt(e)},listen:function(e){return d.appendListener(e)}};return v}},8679:(e,t,n)=>{"use strict";var s=n(59864),r={childContextTypes:!0,contextType:!0,contextTypes:!0,defaultProps:!0,displayName:!0,getDefaultProps:!0,getDerivedStateFromError:!0,getDerivedStateFromProps:!0,mixins:!0,propTypes:!0,type:!0},o={name:!0,length:!0,prototype:!0,caller:!0,callee:!0,arguments:!0,arity:!0},a={$$typeof:!0,compare:!0,defaultProps:!0,displayName:!0,propTypes:!0,type:!0},i={};function c(e){return s.isMemo(e)?a:i[e.$$typeof]||r}i[s.ForwardRef]={$$typeof:!0,render:!0,defaultProps:!0,displayName:!0,propTypes:!0},i[s.Memo]=a;var l=Object.defineProperty,d=Object.getOwnPropertyNames,p=Object.getOwnPropertySymbols,u=Object.getOwnPropertyDescriptor,m=Object.getPrototypeOf,h=Object.prototype;e.exports=function e(t,n,s){if("string"!=typeof n){if(h){var r=m(n);r&&r!==h&&e(t,r,s)}var a=d(n);p&&(a=a.concat(p(n)));for(var i=c(t),g=c(n),f=0;f<a.length;++f){var b=a[f];if(!(o[b]||s&&s[b]||g&&g[b]||i&&i[b])){var y=u(n,b);try{l(t,b,y)}catch(x){}}}}return t}},41143:e=>{"use strict";e.exports=function(e,t,n,s,r,o,a,i){if(!e){var c;if(void 0===t)c=new Error("Minified exception occurred; use the non-minified dev environment for the full error message and additional helpful warnings.");else{var l=[n,s,r,o,a,i],d=0;(c=new Error(t.replace(/%s/g,(function(){return l[d++]})))).name="Invariant Violation"}throw c.framesToPop=1,c}}},5826:e=>{e.exports=Array.isArray||function(e){return"[object Array]"==Object.prototype.toString.call(e)}},32497:(e,t,n)=>{"use strict";n.r(t)},52295:(e,t,n)=>{"use strict";n.r(t)},74865:function(e,t,n){var s,r;s=function(){var e,t,n={version:"0.2.0"},s=n.settings={minimum:.08,easing:"ease",positionUsing:"",speed:200,trickle:!0,trickleRate:.02,trickleSpeed:800,showSpinner:!0,barSelector:'[role="bar"]',spinnerSelector:'[role="spinner"]',parent:"body",template:'<div class="bar" role="bar"><div class="peg"></div></div><div class="spinner" role="spinner"><div class="spinner-icon"></div></div>'};function r(e,t,n){return e<t?t:e>n?n:e}function o(e){return 100*(-1+e)}function a(e,t,n){var r;return(r="translate3d"===s.positionUsing?{transform:"translate3d("+o(e)+"%,0,0)"}:"translate"===s.positionUsing?{transform:"translate("+o(e)+"%,0)"}:{"margin-left":o(e)+"%"}).transition="all "+t+"ms "+n,r}n.configure=function(e){var t,n;for(t in e)void 0!==(n=e[t])&&e.hasOwnProperty(t)&&(s[t]=n);return this},n.status=null,n.set=function(e){var t=n.isStarted();e=r(e,s.minimum,1),n.status=1===e?null:e;var o=n.render(!t),l=o.querySelector(s.barSelector),d=s.speed,p=s.easing;return o.offsetWidth,i((function(t){""===s.positionUsing&&(s.positionUsing=n.getPositioningCSS()),c(l,a(e,d,p)),1===e?(c(o,{transition:"none",opacity:1}),o.offsetWidth,setTimeout((function(){c(o,{transition:"all "+d+"ms linear",opacity:0}),setTimeout((function(){n.remove(),t()}),d)}),d)):setTimeout(t,d)})),this},n.isStarted=function(){return"number"==typeof n.status},n.start=function(){n.status||n.set(0);var e=function(){setTimeout((function(){n.status&&(n.trickle(),e())}),s.trickleSpeed)};return s.trickle&&e(),this},n.done=function(e){return e||n.status?n.inc(.3+.5*Math.random()).set(1):this},n.inc=function(e){var t=n.status;return t?("number"!=typeof e&&(e=(1-t)*r(Math.random()*t,.1,.95)),t=r(t+e,0,.994),n.set(t)):n.start()},n.trickle=function(){return n.inc(Math.random()*s.trickleRate)},e=0,t=0,n.promise=function(s){return s&&"resolved"!==s.state()?(0===t&&n.start(),e++,t++,s.always((function(){0==--t?(e=0,n.done()):n.set((e-t)/e)})),this):this},n.render=function(e){if(n.isRendered())return document.getElementById("nprogress");d(document.documentElement,"nprogress-busy");var t=document.createElement("div");t.id="nprogress",t.innerHTML=s.template;var r,a=t.querySelector(s.barSelector),i=e?"-100":o(n.status||0),l=document.querySelector(s.parent);return c(a,{transition:"all 0 linear",transform:"translate3d("+i+"%,0,0)"}),s.showSpinner||(r=t.querySelector(s.spinnerSelector))&&m(r),l!=document.body&&d(l,"nprogress-custom-parent"),l.appendChild(t),t},n.remove=function(){p(document.documentElement,"nprogress-busy"),p(document.querySelector(s.parent),"nprogress-custom-parent");var e=document.getElementById("nprogress");e&&m(e)},n.isRendered=function(){return!!document.getElementById("nprogress")},n.getPositioningCSS=function(){var e=document.body.style,t="WebkitTransform"in e?"Webkit":"MozTransform"in e?"Moz":"msTransform"in e?"ms":"OTransform"in e?"O":"";return t+"Perspective"in e?"translate3d":t+"Transform"in e?"translate":"margin"};var i=function(){var e=[];function t(){var n=e.shift();n&&n(t)}return function(n){e.push(n),1==e.length&&t()}}(),c=function(){var e=["Webkit","O","Moz","ms"],t={};function n(e){return e.replace(/^-ms-/,"ms-").replace(/-([\da-z])/gi,(function(e,t){return t.toUpperCase()}))}function s(t){var n=document.body.style;if(t in n)return t;for(var s,r=e.length,o=t.charAt(0).toUpperCase()+t.slice(1);r--;)if((s=e[r]+o)in n)return s;return t}function r(e){return e=n(e),t[e]||(t[e]=s(e))}function o(e,t,n){t=r(t),e.style[t]=n}return function(e,t){var n,s,r=arguments;if(2==r.length)for(n in t)void 0!==(s=t[n])&&t.hasOwnProperty(n)&&o(e,n,s);else o(e,r[1],r[2])}}();function l(e,t){return("string"==typeof e?e:u(e)).indexOf(" "+t+" ")>=0}function d(e,t){var n=u(e),s=n+t;l(n,t)||(e.className=s.substring(1))}function p(e,t){var n,s=u(e);l(e,t)&&(n=s.replace(" "+t+" "," "),e.className=n.substring(1,n.length-1))}function u(e){return(" "+(e.className||"")+" ").replace(/\s+/gi," ")}function m(e){e&&e.parentNode&&e.parentNode.removeChild(e)}return n},void 0===(r="function"==typeof s?s.call(t,n,t,e):s)||(e.exports=r)},27418:e=>{"use strict";var t=Object.getOwnPropertySymbols,n=Object.prototype.hasOwnProperty,s=Object.prototype.propertyIsEnumerable;function r(e){if(null==e)throw new TypeError("Object.assign cannot be called with null or undefined");return Object(e)}e.exports=function(){try{if(!Object.assign)return!1;var e=new String("abc");if(e[5]="de","5"===Object.getOwnPropertyNames(e)[0])return!1;for(var t={},n=0;n<10;n++)t["_"+String.fromCharCode(n)]=n;if("0123456789"!==Object.getOwnPropertyNames(t).map((function(e){return t[e]})).join(""))return!1;var s={};return"abcdefghijklmnopqrst".split("").forEach((function(e){s[e]=e})),"abcdefghijklmnopqrst"===Object.keys(Object.assign({},s)).join("")}catch(r){return!1}}()?Object.assign:function(e,o){for(var a,i,c=r(e),l=1;l<arguments.length;l++){for(var d in a=Object(arguments[l]))n.call(a,d)&&(c[d]=a[d]);if(t){i=t(a);for(var p=0;p<i.length;p++)s.call(a,i[p])&&(c[i[p]]=a[i[p]])}}return c}},87410:(e,t,n)=>{"use strict";n.d(t,{Z:()=>o});var s=function(){var e=/(?:^|\s)lang(?:uage)?-([\w-]+)(?=\s|$)/i,t=0,n={},s={util:{encode:function e(t){return t instanceof r?new r(t.type,e(t.content),t.alias):Array.isArray(t)?t.map(e):t.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++t}),e.__id},clone:function e(t,n){var r,o;switch(n=n||{},s.util.type(t)){case"Object":if(o=s.util.objId(t),n[o])return n[o];for(var a in r={},n[o]=r,t)t.hasOwnProperty(a)&&(r[a]=e(t[a],n));return r;case"Array":return o=s.util.objId(t),n[o]?n[o]:(r=[],n[o]=r,t.forEach((function(t,s){r[s]=e(t,n)})),r);default:return t}},getLanguage:function(t){for(;t;){var n=e.exec(t.className);if(n)return n[1].toLowerCase();t=t.parentElement}return"none"},setLanguage:function(t,n){t.className=t.className.replace(RegExp(e,"gi"),""),t.classList.add("language-"+n)},isActive:function(e,t,n){for(var s="no-"+t;e;){var r=e.classList;if(r.contains(t))return!0;if(r.contains(s))return!1;e=e.parentElement}return!!n}},languages:{plain:n,plaintext:n,text:n,txt:n,extend:function(e,t){var n=s.util.clone(s.languages[e]);for(var r in t)n[r]=t[r];return n},insertBefore:function(e,t,n,r){var o=(r=r||s.languages)[e],a={};for(var i in o)if(o.hasOwnProperty(i)){if(i==t)for(var c in n)n.hasOwnProperty(c)&&(a[c]=n[c]);n.hasOwnProperty(i)||(a[i]=o[i])}var l=r[e];return r[e]=a,s.languages.DFS(s.languages,(function(t,n){n===l&&t!=e&&(this[t]=a)})),a},DFS:function e(t,n,r,o){o=o||{};var a=s.util.objId;for(var i in t)if(t.hasOwnProperty(i)){n.call(t,i,t[i],r||i);var c=t[i],l=s.util.type(c);"Object"!==l||o[a(c)]?"Array"!==l||o[a(c)]||(o[a(c)]=!0,e(c,n,i,o)):(o[a(c)]=!0,e(c,n,null,o))}}},plugins:{},highlight:function(e,t,n){var o={code:e,grammar:t,language:n};return s.hooks.run("before-tokenize",o),o.tokens=s.tokenize(o.code,o.grammar),s.hooks.run("after-tokenize",o),r.stringify(s.util.encode(o.tokens),o.language)},tokenize:function(e,t){var n=t.rest;if(n){for(var s in n)t[s]=n[s];delete t.rest}var r=new i;return c(r,r.head,e),a(e,r,t,r.head,0),function(e){var t=[],n=e.head.next;for(;n!==e.tail;)t.push(n.value),n=n.next;return t}(r)},hooks:{all:{},add:function(e,t){var n=s.hooks.all;n[e]=n[e]||[],n[e].push(t)},run:function(e,t){var n=s.hooks.all[e];if(n&&n.length)for(var r,o=0;r=n[o++];)r(t)}},Token:r};function r(e,t,n,s){this.type=e,this.content=t,this.alias=n,this.length=0|(s||"").length}function o(e,t,n,s){e.lastIndex=t;var r=e.exec(n);if(r&&s&&r[1]){var o=r[1].length;r.index+=o,r[0]=r[0].slice(o)}return r}function a(e,t,n,i,d,p){for(var u in n)if(n.hasOwnProperty(u)&&n[u]){var m=n[u];m=Array.isArray(m)?m:[m];for(var h=0;h<m.length;++h){if(p&&p.cause==u+","+h)return;var g=m[h],f=g.inside,b=!!g.lookbehind,y=!!g.greedy,x=g.alias;if(y&&!g.pattern.global){var v=g.pattern.toString().match(/[imsuy]*$/)[0];g.pattern=RegExp(g.pattern.source,v+"g")}for(var A=g.pattern||g,S=i.next,E=d;S!==t.tail&&!(p&&E>=p.reach);E+=S.value.length,S=S.next){var L=S.value;if(t.length>e.length)return;if(!(L instanceof r)){var M,k=1;if(y){if(!(M=o(A,E,e,b))||M.index>=e.length)break;var _=M.index,w=M.index+M[0].length,I=E;for(I+=S.value.length;_>=I;)I+=(S=S.next).value.length;if(E=I-=S.value.length,S.value instanceof r)continue;for(var C=S;C!==t.tail&&(I<w||"string"==typeof C.value);C=C.next)k++,I+=C.value.length;k--,L=e.slice(E,I),M.index-=E}else if(!(M=o(A,0,L,b)))continue;_=M.index;var Q=M[0],R=L.slice(0,_),D=L.slice(_+Q.length),T=E+L.length;p&&T>p.reach&&(p.reach=T);var O=S.prev;if(R&&(O=c(t,O,R),E+=R.length),l(t,O,k),S=c(t,O,new r(u,f?s.tokenize(Q,f):Q,x,Q)),D&&c(t,S,D),k>1){var P={cause:u+","+h,reach:T};a(e,t,n,S.prev,E,P),p&&P.reach>p.reach&&(p.reach=P.reach)}}}}}}function i(){var e={value:null,prev:null,next:null},t={value:null,prev:e,next:null};e.next=t,this.head=e,this.tail=t,this.length=0}function c(e,t,n){var s=t.next,r={value:n,prev:t,next:s};return t.next=r,s.prev=r,e.length++,r}function l(e,t,n){for(var s=t.next,r=0;r<n&&s!==e.tail;r++)s=s.next;t.next=s,s.prev=t,e.length-=r}return r.stringify=function e(t,n){if("string"==typeof t)return t;if(Array.isArray(t)){var r="";return t.forEach((function(t){r+=e(t,n)})),r}var o={type:t.type,content:e(t.content,n),tag:"span",classes:["token",t.type],attributes:{},language:n},a=t.alias;a&&(Array.isArray(a)?Array.prototype.push.apply(o.classes,a):o.classes.push(a)),s.hooks.run("wrap",o);var i="";for(var c in o.attributes)i+=" "+c+'="'+(o.attributes[c]||"").replace(/"/g,"&quot;")+'"';return"<"+o.tag+' class="'+o.classes.join(" ")+'"'+i+">"+o.content+"</"+o.tag+">"},s}(),r=s;s.default=s,r.languages.markup={comment:{pattern:/<!--(?:(?!<!--)[\s\S])*?-->/,greedy:!0},prolog:{pattern:/<\?[\s\S]+?\?>/,greedy:!0},doctype:{pattern:/<!DOCTYPE(?:[^>"'[\]]|"[^"]*"|'[^']*')+(?:\[(?:[^<"'\]]|"[^"]*"|'[^']*'|<(?!!--)|<!--(?:[^-]|-(?!->))*-->)*\]\s*)?>/i,greedy:!0,inside:{"internal-subset":{pattern:/(^[^\[]*\[)[\s\S]+(?=\]>$)/,lookbehind:!0,greedy:!0,inside:null},string:{pattern:/"[^"]*"|'[^']*'/,greedy:!0},punctuation:/^<!|>$|[[\]]/,"doctype-tag":/^DOCTYPE/i,name:/[^\s<>'"]+/}},cdata:{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,greedy:!0},tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s/>])))+)?\s*\/?>/,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"special-attr":[],"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/,inside:{punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:[{pattern:/&[\da-z]{1,8};/i,alias:"named-entity"},/&#x?[\da-f]{1,8};/i]},r.languages.markup.tag.inside["attr-value"].inside.entity=r.languages.markup.entity,r.languages.markup.doctype.inside["internal-subset"].inside=r.languages.markup,r.hooks.add("wrap",(function(e){"entity"===e.type&&(e.attributes.title=e.content.replace(/&amp;/,"&"))})),Object.defineProperty(r.languages.markup.tag,"addInlined",{value:function(e,t){var n={};n["language-"+t]={pattern:/(^<!\[CDATA\[)[\s\S]+?(?=\]\]>$)/i,lookbehind:!0,inside:r.languages[t]},n.cdata=/^<!\[CDATA\[|\]\]>$/i;var s={"included-cdata":{pattern:/<!\[CDATA\[[\s\S]*?\]\]>/i,inside:n}};s["language-"+t]={pattern:/[\s\S]+/,inside:r.languages[t]};var o={};o[e]={pattern:RegExp(/(<__[^>]*>)(?:<!\[CDATA\[(?:[^\]]|\](?!\]>))*\]\]>|(?!<!\[CDATA\[)[\s\S])*?(?=<\/__>)/.source.replace(/__/g,(function(){return e})),"i"),lookbehind:!0,greedy:!0,inside:s},r.languages.insertBefore("markup","cdata",o)}}),Object.defineProperty(r.languages.markup.tag,"addAttribute",{value:function(e,t){r.languages.markup.tag.inside["special-attr"].push({pattern:RegExp(/(^|["'\s])/.source+"(?:"+e+")"+/\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))/.source,"i"),lookbehind:!0,inside:{"attr-name":/^[^\s=]+/,"attr-value":{pattern:/=[\s\S]+/,inside:{value:{pattern:/(^=\s*(["']|(?!["'])))\S[\s\S]*(?=\2$)/,lookbehind:!0,alias:[t,"language-"+t],inside:r.languages[t]},punctuation:[{pattern:/^=/,alias:"attr-equals"},/"|'/]}}}})}}),r.languages.html=r.languages.markup,r.languages.mathml=r.languages.markup,r.languages.svg=r.languages.markup,r.languages.xml=r.languages.extend("markup",{}),r.languages.ssml=r.languages.xml,r.languages.atom=r.languages.xml,r.languages.rss=r.languages.xml,function(e){var t="\\b(?:BASH|BASHOPTS|BASH_ALIASES|BASH_ARGC|BASH_ARGV|BASH_CMDS|BASH_COMPLETION_COMPAT_DIR|BASH_LINENO|BASH_REMATCH|BASH_SOURCE|BASH_VERSINFO|BASH_VERSION|COLORTERM|COLUMNS|COMP_WORDBREAKS|DBUS_SESSION_BUS_ADDRESS|DEFAULTS_PATH|DESKTOP_SESSION|DIRSTACK|DISPLAY|EUID|GDMSESSION|GDM_LANG|GNOME_KEYRING_CONTROL|GNOME_KEYRING_PID|GPG_AGENT_INFO|GROUPS|HISTCONTROL|HISTFILE|HISTFILESIZE|HISTSIZE|HOME|HOSTNAME|HOSTTYPE|IFS|INSTANCE|JOB|LANG|LANGUAGE|LC_ADDRESS|LC_ALL|LC_IDENTIFICATION|LC_MEASUREMENT|LC_MONETARY|LC_NAME|LC_NUMERIC|LC_PAPER|LC_TELEPHONE|LC_TIME|LESSCLOSE|LESSOPEN|LINES|LOGNAME|LS_COLORS|MACHTYPE|MAILCHECK|MANDATORY_PATH|NO_AT_BRIDGE|OLDPWD|OPTERR|OPTIND|ORBIT_SOCKETDIR|OSTYPE|PAPERSIZE|PATH|PIPESTATUS|PPID|PS1|PS2|PS3|PS4|PWD|RANDOM|REPLY|SECONDS|SELINUX_INIT|SESSION|SESSIONTYPE|SESSION_MANAGER|SHELL|SHELLOPTS|SHLVL|SSH_AUTH_SOCK|TERM|UID|UPSTART_EVENTS|UPSTART_INSTANCE|UPSTART_JOB|UPSTART_SESSION|USER|WINDOWID|XAUTHORITY|XDG_CONFIG_DIRS|XDG_CURRENT_DESKTOP|XDG_DATA_DIRS|XDG_GREETER_DATA_DIR|XDG_MENU_PREFIX|XDG_RUNTIME_DIR|XDG_SEAT|XDG_SEAT_PATH|XDG_SESSION_DESKTOP|XDG_SESSION_ID|XDG_SESSION_PATH|XDG_SESSION_TYPE|XDG_VTNR|XMODIFIERS)\\b",n={pattern:/(^(["']?)\w+\2)[ \t]+\S.*/,lookbehind:!0,alias:"punctuation",inside:null},s={bash:n,environment:{pattern:RegExp("\\$"+t),alias:"constant"},variable:[{pattern:/\$?\(\([\s\S]+?\)\)/,greedy:!0,inside:{variable:[{pattern:/(^\$\(\([\s\S]+)\)\)/,lookbehind:!0},/^\$\(\(/],number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee]-?\d+)?/,operator:/--|\+\+|\*\*=?|<<=?|>>=?|&&|\|\||[=!+\-*/%<>^&|]=?|[?~:]/,punctuation:/\(\(?|\)\)?|,|;/}},{pattern:/\$\((?:\([^)]+\)|[^()])+\)|`[^`]+`/,greedy:!0,inside:{variable:/^\$\(|^`|\)$|`$/}},{pattern:/\$\{[^}]+\}/,greedy:!0,inside:{operator:/:[-=?+]?|[!\/]|##?|%%?|\^\^?|,,?/,punctuation:/[\[\]]/,environment:{pattern:RegExp("(\\{)"+t),lookbehind:!0,alias:"constant"}}},/\$(?:\w+|[#?*!@$])/],entity:/\\(?:[abceEfnrtv\\"]|O?[0-7]{1,3}|U[0-9a-fA-F]{8}|u[0-9a-fA-F]{4}|x[0-9a-fA-F]{1,2})/};e.languages.bash={shebang:{pattern:/^#!\s*\/.*/,alias:"important"},comment:{pattern:/(^|[^"{\\$])#.*/,lookbehind:!0},"function-name":[{pattern:/(\bfunction\s+)[\w-]+(?=(?:\s*\(?:\s*\))?\s*\{)/,lookbehind:!0,alias:"function"},{pattern:/\b[\w-]+(?=\s*\(\s*\)\s*\{)/,alias:"function"}],"for-or-select":{pattern:/(\b(?:for|select)\s+)\w+(?=\s+in\s)/,alias:"variable",lookbehind:!0},"assign-left":{pattern:/(^|[\s;|&]|[<>]\()\w+(?=\+?=)/,inside:{environment:{pattern:RegExp("(^|[\\s;|&]|[<>]\\()"+t),lookbehind:!0,alias:"constant"}},alias:"variable",lookbehind:!0},string:[{pattern:/((?:^|[^<])<<-?\s*)(\w+)\s[\s\S]*?(?:\r?\n|\r)\2/,lookbehind:!0,greedy:!0,inside:s},{pattern:/((?:^|[^<])<<-?\s*)(["'])(\w+)\2\s[\s\S]*?(?:\r?\n|\r)\3/,lookbehind:!0,greedy:!0,inside:{bash:n}},{pattern:/(^|[^\\](?:\\\\)*)"(?:\\[\s\S]|\$\([^)]+\)|\$(?!\()|`[^`]+`|[^"\\`$])*"/,lookbehind:!0,greedy:!0,inside:s},{pattern:/(^|[^$\\])'[^']*'/,lookbehind:!0,greedy:!0},{pattern:/\$'(?:[^'\\]|\\[\s\S])*'/,greedy:!0,inside:{entity:s.entity}}],environment:{pattern:RegExp("\\$?"+t),alias:"constant"},variable:s.variable,function:{pattern:/(^|[\s;|&]|[<>]\()(?:add|apropos|apt|apt-cache|apt-get|aptitude|aspell|automysqlbackup|awk|basename|bash|bc|bconsole|bg|bzip2|cal|cat|cfdisk|chgrp|chkconfig|chmod|chown|chroot|cksum|clear|cmp|column|comm|composer|cp|cron|crontab|csplit|curl|cut|date|dc|dd|ddrescue|debootstrap|df|diff|diff3|dig|dir|dircolors|dirname|dirs|dmesg|docker|docker-compose|du|egrep|eject|env|ethtool|expand|expect|expr|fdformat|fdisk|fg|fgrep|file|find|fmt|fold|format|free|fsck|ftp|fuser|gawk|git|gparted|grep|groupadd|groupdel|groupmod|groups|grub-mkconfig|gzip|halt|head|hg|history|host|hostname|htop|iconv|id|ifconfig|ifdown|ifup|import|install|ip|jobs|join|kill|killall|less|link|ln|locate|logname|logrotate|look|lpc|lpr|lprint|lprintd|lprintq|lprm|ls|lsof|lynx|make|man|mc|mdadm|mkconfig|mkdir|mke2fs|mkfifo|mkfs|mkisofs|mknod|mkswap|mmv|more|most|mount|mtools|mtr|mutt|mv|nano|nc|netstat|nice|nl|node|nohup|notify-send|npm|nslookup|op|open|parted|passwd|paste|pathchk|ping|pkill|pnpm|podman|podman-compose|popd|pr|printcap|printenv|ps|pushd|pv|quota|quotacheck|quotactl|ram|rar|rcp|reboot|remsync|rename|renice|rev|rm|rmdir|rpm|rsync|scp|screen|sdiff|sed|sendmail|seq|service|sftp|sh|shellcheck|shuf|shutdown|sleep|slocate|sort|split|ssh|stat|strace|su|sudo|sum|suspend|swapon|sync|tac|tail|tar|tee|time|timeout|top|touch|tr|traceroute|tsort|tty|umount|uname|unexpand|uniq|units|unrar|unshar|unzip|update-grub|uptime|useradd|userdel|usermod|users|uudecode|uuencode|v|vcpkg|vdir|vi|vim|virsh|vmstat|wait|watch|wc|wget|whereis|which|who|whoami|write|xargs|xdg-open|yarn|yes|zenity|zip|zsh|zypper)(?=$|[)\s;|&])/,lookbehind:!0},keyword:{pattern:/(^|[\s;|&]|[<>]\()(?:case|do|done|elif|else|esac|fi|for|function|if|in|select|then|until|while)(?=$|[)\s;|&])/,lookbehind:!0},builtin:{pattern:/(^|[\s;|&]|[<>]\()(?:\.|:|alias|bind|break|builtin|caller|cd|command|continue|declare|echo|enable|eval|exec|exit|export|getopts|hash|help|let|local|logout|mapfile|printf|pwd|read|readarray|readonly|return|set|shift|shopt|source|test|times|trap|type|typeset|ulimit|umask|unalias|unset)(?=$|[)\s;|&])/,lookbehind:!0,alias:"class-name"},boolean:{pattern:/(^|[\s;|&]|[<>]\()(?:false|true)(?=$|[)\s;|&])/,lookbehind:!0},"file-descriptor":{pattern:/\B&\d\b/,alias:"important"},operator:{pattern:/\d?<>|>\||\+=|=[=~]?|!=?|<<[<-]?|[&\d]?>>|\d[<>]&?|[<>][&=]?|&[>&]?|\|[&|]?/,inside:{"file-descriptor":{pattern:/^\d/,alias:"important"}}},punctuation:/\$?\(\(?|\)\)?|\.\.|[{}[\];\\]/,number:{pattern:/(^|\s)(?:[1-9]\d*|0)(?:[.,]\d+)?\b/,lookbehind:!0}},n.inside=e.languages.bash;for(var r=["comment","function-name","for-or-select","assign-left","string","environment","function","keyword","builtin","boolean","file-descriptor","operator","punctuation","number"],o=s.variable[1].inside,a=0;a<r.length;a++)o[r[a]]=e.languages.bash[r[a]];e.languages.shell=e.languages.bash}(r),r.languages.clike={comment:[{pattern:/(^|[^\\])\/\*[\s\S]*?(?:\*\/|$)/,lookbehind:!0,greedy:!0},{pattern:/(^|[^\\:])\/\/.*/,lookbehind:!0,greedy:!0}],string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|trait)\s+|\bcatch\s+\()[\w.\\]+/i,lookbehind:!0,inside:{punctuation:/[.\\]/}},keyword:/\b(?:break|catch|continue|do|else|finally|for|function|if|in|instanceof|new|null|return|throw|try|while)\b/,boolean:/\b(?:false|true)\b/,function:/\b\w+(?=\()/,number:/\b0x[\da-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,operator:/[<>]=?|[!=]=?=?|--?|\+\+?|&&?|\|\|?|[?*/~^%]/,punctuation:/[{}[\];(),.:]/},r.languages.c=r.languages.extend("clike",{comment:{pattern:/\/\/(?:[^\r\n\\]|\\(?:\r\n?|\n|(?![\r\n])))*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},"class-name":{pattern:/(\b(?:enum|struct)\s+(?:__attribute__\s*\(\([\s\S]*?\)\)\s*)?)\w+|\b[a-z]\w*_t\b/,lookbehind:!0},keyword:/\b(?:_Alignas|_Alignof|_Atomic|_Bool|_Complex|_Generic|_Imaginary|_Noreturn|_Static_assert|_Thread_local|__attribute__|asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|inline|int|long|register|return|short|signed|sizeof|static|struct|switch|typedef|typeof|union|unsigned|void|volatile|while)\b/,function:/\b[a-z_]\w*(?=\s*\()/i,number:/(?:\b0x(?:[\da-f]+(?:\.[\da-f]*)?|\.[\da-f]+)(?:p[+-]?\d+)?|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?)[ful]{0,4}/i,operator:/>>=?|<<=?|->|([-+&|:])\1|[?:~]|[-+*/%&|^!=<>]=?/}),r.languages.insertBefore("c","string",{char:{pattern:/'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n]){0,32}'/,greedy:!0}}),r.languages.insertBefore("c","string",{macro:{pattern:/(^[\t ]*)#\s*[a-z](?:[^\r\n\\/]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|\\(?:\r\n|[\s\S]))*/im,lookbehind:!0,greedy:!0,alias:"property",inside:{string:[{pattern:/^(#\s*include\s*)<[^>]+>/,lookbehind:!0},r.languages.c.string],char:r.languages.c.char,comment:r.languages.c.comment,"macro-name":[{pattern:/(^#\s*define\s+)\w+\b(?!\()/i,lookbehind:!0},{pattern:/(^#\s*define\s+)\w+\b(?=\()/i,lookbehind:!0,alias:"function"}],directive:{pattern:/^(#\s*)[a-z]+/,lookbehind:!0,alias:"keyword"},"directive-hash":/^#/,punctuation:/##|\\(?=[\r\n])/,expression:{pattern:/\S[\s\S]*/,inside:r.languages.c}}}}),r.languages.insertBefore("c","function",{constant:/\b(?:EOF|NULL|SEEK_CUR|SEEK_END|SEEK_SET|__DATE__|__FILE__|__LINE__|__TIMESTAMP__|__TIME__|__func__|stderr|stdin|stdout)\b/}),delete r.languages.c.boolean,function(e){var t=/\b(?:alignas|alignof|asm|auto|bool|break|case|catch|char|char16_t|char32_t|char8_t|class|co_await|co_return|co_yield|compl|concept|const|const_cast|consteval|constexpr|constinit|continue|decltype|default|delete|do|double|dynamic_cast|else|enum|explicit|export|extern|final|float|for|friend|goto|if|import|inline|int|int16_t|int32_t|int64_t|int8_t|long|module|mutable|namespace|new|noexcept|nullptr|operator|override|private|protected|public|register|reinterpret_cast|requires|return|short|signed|sizeof|static|static_assert|static_cast|struct|switch|template|this|thread_local|throw|try|typedef|typeid|typename|uint16_t|uint32_t|uint64_t|uint8_t|union|unsigned|using|virtual|void|volatile|wchar_t|while)\b/,n=/\b(?!<keyword>)\w+(?:\s*\.\s*\w+)*\b/.source.replace(/<keyword>/g,(function(){return t.source}));e.languages.cpp=e.languages.extend("c",{"class-name":[{pattern:RegExp(/(\b(?:class|concept|enum|struct|typename)\s+)(?!<keyword>)\w+/.source.replace(/<keyword>/g,(function(){return t.source}))),lookbehind:!0},/\b[A-Z]\w*(?=\s*::\s*\w+\s*\()/,/\b[A-Z_]\w*(?=\s*::\s*~\w+\s*\()/i,/\b\w+(?=\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>\s*::\s*\w+\s*\()/],keyword:t,number:{pattern:/(?:\b0b[01']+|\b0x(?:[\da-f']+(?:\.[\da-f']*)?|\.[\da-f']+)(?:p[+-]?[\d']+)?|(?:\b[\d']+(?:\.[\d']*)?|\B\.[\d']+)(?:e[+-]?[\d']+)?)[ful]{0,4}/i,greedy:!0},operator:/>>=?|<<=?|->|--|\+\+|&&|\|\||[?:~]|<=>|[-+*/%&|^!=<>]=?|\b(?:and|and_eq|bitand|bitor|not|not_eq|or|or_eq|xor|xor_eq)\b/,boolean:/\b(?:false|true)\b/}),e.languages.insertBefore("cpp","string",{module:{pattern:RegExp(/(\b(?:import|module)\s+)/.source+"(?:"+/"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|<[^<>\r\n]*>/.source+"|"+/<mod-name>(?:\s*:\s*<mod-name>)?|:\s*<mod-name>/.source.replace(/<mod-name>/g,(function(){return n}))+")"),lookbehind:!0,greedy:!0,inside:{string:/^[<"][\s\S]+/,operator:/:/,punctuation:/\./}},"raw-string":{pattern:/R"([^()\\ ]{0,16})\([\s\S]*?\)\1"/,alias:"string",greedy:!0}}),e.languages.insertBefore("cpp","keyword",{"generic-function":{pattern:/\b(?!operator\b)[a-z_]\w*\s*<(?:[^<>]|<[^<>]*>)*>(?=\s*\()/i,inside:{function:/^\w+/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:e.languages.cpp}}}}),e.languages.insertBefore("cpp","operator",{"double-colon":{pattern:/::/,alias:"punctuation"}}),e.languages.insertBefore("cpp","class-name",{"base-clause":{pattern:/(\b(?:class|struct)\s+\w+\s*:\s*)[^;{}"'\s]+(?:\s+[^;{}"'\s]+)*(?=\s*[;{])/,lookbehind:!0,greedy:!0,inside:e.languages.extend("cpp",{})}}),e.languages.insertBefore("inside","double-colon",{"class-name":/\b[a-z_]\w*\b(?!\s*::)/i},e.languages.cpp["base-clause"])}(r),function(e){var t=/(?:"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"|'(?:\\(?:\r\n|[\s\S])|[^'\\\r\n])*')/;e.languages.css={comment:/\/\*[\s\S]*?\*\//,atrule:{pattern:/@[\w-](?:[^;{\s]|\s+(?![\s{]))*(?:;|(?=\s*\{))/,inside:{rule:/^@[\w-]+/,"selector-function-argument":{pattern:/(\bselector\s*\(\s*(?![\s)]))(?:[^()\s]|\s+(?![\s)])|\((?:[^()]|\([^()]*\))*\))+(?=\s*\))/,lookbehind:!0,alias:"selector"},keyword:{pattern:/(^|[^\w-])(?:and|not|only|or)(?![\w-])/,lookbehind:!0}}},url:{pattern:RegExp("\\burl\\((?:"+t.source+"|"+/(?:[^\\\r\n()"']|\\[\s\S])*/.source+")\\)","i"),greedy:!0,inside:{function:/^url/i,punctuation:/^\(|\)$/,string:{pattern:RegExp("^"+t.source+"$"),alias:"url"}}},selector:{pattern:RegExp("(^|[{}\\s])[^{}\\s](?:[^{};\"'\\s]|\\s+(?![\\s{])|"+t.source+")*(?=\\s*\\{)"),lookbehind:!0},string:{pattern:t,greedy:!0},property:{pattern:/(^|[^-\w\xA0-\uFFFF])(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*(?=\s*:)/i,lookbehind:!0},important:/!important\b/i,function:{pattern:/(^|[^-a-z0-9])[-a-z0-9]+(?=\()/i,lookbehind:!0},punctuation:/[(){};:,]/},e.languages.css.atrule.inside.rest=e.languages.css;var n=e.languages.markup;n&&(n.tag.addInlined("style","css"),n.tag.addAttribute("style","css"))}(r),function(e){var t,n=/("|')(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/;e.languages.css.selector={pattern:e.languages.css.selector.pattern,lookbehind:!0,inside:t={"pseudo-element":/:(?:after|before|first-letter|first-line|selection)|::[-\w]+/,"pseudo-class":/:[-\w]+/,class:/\.[-\w]+/,id:/#[-\w]+/,attribute:{pattern:RegExp("\\[(?:[^[\\]\"']|"+n.source+")*\\]"),greedy:!0,inside:{punctuation:/^\[|\]$/,"case-sensitivity":{pattern:/(\s)[si]$/i,lookbehind:!0,alias:"keyword"},namespace:{pattern:/^(\s*)(?:(?!\s)[-*\w\xA0-\uFFFF])*\|(?!=)/,lookbehind:!0,inside:{punctuation:/\|$/}},"attr-name":{pattern:/^(\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+/,lookbehind:!0},"attr-value":[n,{pattern:/(=\s*)(?:(?!\s)[-\w\xA0-\uFFFF])+(?=\s*$)/,lookbehind:!0}],operator:/[|~*^$]?=/}},"n-th":[{pattern:/(\(\s*)[+-]?\d*[\dn](?:\s*[+-]\s*\d+)?(?=\s*\))/,lookbehind:!0,inside:{number:/[\dn]+/,operator:/[+-]/}},{pattern:/(\(\s*)(?:even|odd)(?=\s*\))/i,lookbehind:!0}],combinator:/>|\+|~|\|\|/,punctuation:/[(),]/}},e.languages.css.atrule.inside["selector-function-argument"].inside=t,e.languages.insertBefore("css","property",{variable:{pattern:/(^|[^-\w\xA0-\uFFFF])--(?!\s)[-_a-z\xA0-\uFFFF](?:(?!\s)[-\w\xA0-\uFFFF])*/i,lookbehind:!0}});var s={pattern:/(\b\d+)(?:%|[a-z]+(?![\w-]))/,lookbehind:!0},r={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0};e.languages.insertBefore("css","function",{operator:{pattern:/(\s)[+\-*\/](?=\s)/,lookbehind:!0},hexcode:{pattern:/\B#[\da-f]{3,8}\b/i,alias:"color"},color:[{pattern:/(^|[^\w-])(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)(?![\w-])/i,lookbehind:!0},{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:s,number:r,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:s,number:r})}(r),r.languages.javascript=r.languages.extend("clike",{"class-name":[r.languages.clike["class-name"],{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$A-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\.(?:constructor|prototype))/,lookbehind:!0}],keyword:[{pattern:/((?:^|\})\s*)catch\b/,lookbehind:!0},{pattern:/(^|[^.]|\.\.\.\s*)\b(?:as|assert(?=\s*\{)|async(?=\s*(?:function\b|\(|[$\w\xA0-\uFFFF]|$))|await|break|case|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally(?=\s*(?:\{|$))|for|from(?=\s*(?:['"]|$))|function|(?:get|set)(?=\s*(?:[#\[$\w\xA0-\uFFFF]|$))|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)\b/,lookbehind:!0}],function:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*(?:\.\s*(?:apply|bind|call)\s*)?\()/,number:{pattern:RegExp(/(^|[^\w$])/.source+"(?:"+/NaN|Infinity/.source+"|"+/0[bB][01]+(?:_[01]+)*n?/.source+"|"+/0[oO][0-7]+(?:_[0-7]+)*n?/.source+"|"+/0[xX][\dA-Fa-f]+(?:_[\dA-Fa-f]+)*n?/.source+"|"+/\d+(?:_\d+)*n/.source+"|"+/(?:\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\.\d+(?:_\d+)*)(?:[Ee][+-]?\d+(?:_\d+)*)?/.source+")"+/(?![\w$])/.source),lookbehind:!0},operator:/--|\+\+|\*\*=?|=>|&&=?|\|\|=?|[!=]==|<<=?|>>>?=?|[-+*/%&|^!=<>]=?|\.{3}|\?\?=?|\?\.?|[~:]/}),r.languages.javascript["class-name"][0].pattern=/(\b(?:class|extends|implements|instanceof|interface|new)\s+)[\w.\\]+/,r.languages.insertBefore("javascript","keyword",{regex:{pattern:/((?:^|[^$\w\xA0-\uFFFF."'\])\s]|\b(?:return|yield))\s*)\/(?:\[(?:[^\]\\\r\n]|\\.)*\]|\\.|[^/\\\[\r\n])+\/[dgimyus]{0,7}(?=(?:\s|\/\*(?:[^*]|\*(?!\/))*\*\/)*(?:$|[\r\n,.;:})\]]|\/\/))/,lookbehind:!0,greedy:!0,inside:{"regex-source":{pattern:/^(\/)[\s\S]+(?=\/[a-z]*$)/,lookbehind:!0,alias:"language-regex",inside:r.languages.regex},"regex-delimiter":/^\/|\/$/,"regex-flags":/^[a-z]+$/}},"function-variable":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*[=:]\s*(?:async\s*)?(?:\bfunction\b|(?:\((?:[^()]|\([^()]*\))*\)|(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)\s*=>))/,alias:"function"},parameter:[{pattern:/(function(?:\s+(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*)?\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\))/,lookbehind:!0,inside:r.languages.javascript},{pattern:/(^|[^$\w\xA0-\uFFFF])(?!\s)[_$a-z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*=>)/i,lookbehind:!0,inside:r.languages.javascript},{pattern:/(\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*=>)/,lookbehind:!0,inside:r.languages.javascript},{pattern:/((?:\b|\s|^)(?!(?:as|async|await|break|case|catch|class|const|continue|debugger|default|delete|do|else|enum|export|extends|finally|for|from|function|get|if|implements|import|in|instanceof|interface|let|new|null|of|package|private|protected|public|return|set|static|super|switch|this|throw|try|typeof|undefined|var|void|while|with|yield)(?![$\w\xA0-\uFFFF]))(?:(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*)\(\s*|\]\s*\(\s*)(?!\s)(?:[^()\s]|\s+(?![\s)])|\([^()]*\))+(?=\s*\)\s*\{)/,lookbehind:!0,inside:r.languages.javascript}],constant:/\b[A-Z](?:[A-Z_]|\dx?)*\b/}),r.languages.insertBefore("javascript","string",{hashbang:{pattern:/^#!.*/,greedy:!0,alias:"comment"},"template-string":{pattern:/`(?:\\[\s\S]|\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}|(?!\$\{)[^\\`])*`/,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},interpolation:{pattern:/((?:^|[^\\])(?:\\{2})*)\$\{(?:[^{}]|\{(?:[^{}]|\{[^}]*\})*\})+\}/,lookbehind:!0,inside:{"interpolation-punctuation":{pattern:/^\$\{|\}$/,alias:"punctuation"},rest:r.languages.javascript}},string:/[\s\S]+/}},"string-property":{pattern:/((?:^|[,{])[ \t]*)(["'])(?:\\(?:\r\n|[\s\S])|(?!\2)[^\\\r\n])*\2(?=\s*:)/m,lookbehind:!0,greedy:!0,alias:"property"}}),r.languages.insertBefore("javascript","operator",{"literal-property":{pattern:/((?:^|[,{])[ \t]*)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?=\s*:)/m,lookbehind:!0,alias:"property"}}),r.languages.markup&&(r.languages.markup.tag.addInlined("script","javascript"),r.languages.markup.tag.addAttribute(/on(?:abort|blur|change|click|composition(?:end|start|update)|dblclick|error|focus(?:in|out)?|key(?:down|up)|load|mouse(?:down|enter|leave|move|out|over|up)|reset|resize|scroll|select|slotchange|submit|unload|wheel)/.source,"javascript")),r.languages.js=r.languages.javascript,function(e){var t=/#(?!\{).+/,n={pattern:/#\{[^}]+\}/,alias:"variable"};e.languages.coffeescript=e.languages.extend("javascript",{comment:t,string:[{pattern:/'(?:\\[\s\S]|[^\\'])*'/,greedy:!0},{pattern:/"(?:\\[\s\S]|[^\\"])*"/,greedy:!0,inside:{interpolation:n}}],keyword:/\b(?:and|break|by|catch|class|continue|debugger|delete|do|each|else|extend|extends|false|finally|for|if|in|instanceof|is|isnt|let|loop|namespace|new|no|not|null|of|off|on|or|own|return|super|switch|then|this|throw|true|try|typeof|undefined|unless|until|when|while|window|with|yes|yield)\b/,"class-member":{pattern:/@(?!\d)\w+/,alias:"variable"}}),e.languages.insertBefore("coffeescript","comment",{"multiline-comment":{pattern:/###[\s\S]+?###/,alias:"comment"},"block-regex":{pattern:/\/{3}[\s\S]*?\/{3}/,alias:"regex",inside:{comment:t,interpolation:n}}}),e.languages.insertBefore("coffeescript","string",{"inline-javascript":{pattern:/`(?:\\[\s\S]|[^\\`])*`/,inside:{delimiter:{pattern:/^`|`$/,alias:"punctuation"},script:{pattern:/[\s\S]+/,alias:"language-javascript",inside:e.languages.javascript}}},"multiline-string":[{pattern:/'''[\s\S]*?'''/,greedy:!0,alias:"string"},{pattern:/"""[\s\S]*?"""/,greedy:!0,alias:"string",inside:{interpolation:n}}]}),e.languages.insertBefore("coffeescript","keyword",{property:/(?!\d)\w+(?=\s*:(?!:))/}),delete e.languages.coffeescript["template-string"],e.languages.coffee=e.languages.coffeescript}(r),function(e){var t=/[*&][^\s[\]{},]+/,n=/!(?:<[\w\-%#;/?:@&=+$,.!~*'()[\]]+>|(?:[a-zA-Z\d-]*!)?[\w\-%#;/?:@&=+$.~*'()]+)?/,s="(?:"+n.source+"(?:[ \t]+"+t.source+")?|"+t.source+"(?:[ \t]+"+n.source+")?)",r=/(?:[^\s\x00-\x08\x0e-\x1f!"#%&'*,\-:>?@[\]`{|}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]|[?:-]<PLAIN>)(?:[ \t]*(?:(?![#:])<PLAIN>|:<PLAIN>))*/.source.replace(/<PLAIN>/g,(function(){return/[^\s\x00-\x08\x0e-\x1f,[\]{}\x7f-\x84\x86-\x9f\ud800-\udfff\ufffe\uffff]/.source})),o=/"(?:[^"\\\r\n]|\\.)*"|'(?:[^'\\\r\n]|\\.)*'/.source;function a(e,t){t=(t||"").replace(/m/g,"")+"m";var n=/([:\-,[{]\s*(?:\s<<prop>>[ \t]+)?)(?:<<value>>)(?=[ \t]*(?:$|,|\]|\}|(?:[\r\n]\s*)?#))/.source.replace(/<<prop>>/g,(function(){return s})).replace(/<<value>>/g,(function(){return e}));return RegExp(n,t)}e.languages.yaml={scalar:{pattern:RegExp(/([\-:]\s*(?:\s<<prop>>[ \t]+)?[|>])[ \t]*(?:((?:\r?\n|\r)[ \t]+)\S[^\r\n]*(?:\2[^\r\n]+)*)/.source.replace(/<<prop>>/g,(function(){return s}))),lookbehind:!0,alias:"string"},comment:/#.*/,key:{pattern:RegExp(/((?:^|[:\-,[{\r\n?])[ \t]*(?:<<prop>>[ \t]+)?)<<key>>(?=\s*:\s)/.source.replace(/<<prop>>/g,(function(){return s})).replace(/<<key>>/g,(function(){return"(?:"+r+"|"+o+")"}))),lookbehind:!0,greedy:!0,alias:"atrule"},directive:{pattern:/(^[ \t]*)%.+/m,lookbehind:!0,alias:"important"},datetime:{pattern:a(/\d{4}-\d\d?-\d\d?(?:[tT]|[ \t]+)\d\d?:\d{2}:\d{2}(?:\.\d*)?(?:[ \t]*(?:Z|[-+]\d\d?(?::\d{2})?))?|\d{4}-\d{2}-\d{2}|\d\d?:\d{2}(?::\d{2}(?:\.\d*)?)?/.source),lookbehind:!0,alias:"number"},boolean:{pattern:a(/false|true/.source,"i"),lookbehind:!0,alias:"important"},null:{pattern:a(/null|~/.source,"i"),lookbehind:!0,alias:"important"},string:{pattern:a(o),lookbehind:!0,greedy:!0},number:{pattern:a(/[+-]?(?:0x[\da-f]+|0o[0-7]+|(?:\d+(?:\.\d*)?|\.\d+)(?:e[+-]?\d+)?|\.inf|\.nan)/.source,"i"),lookbehind:!0},tag:n,important:t,punctuation:/---|[:[\]{}\-,|>?]|\.\.\./},e.languages.yml=e.languages.yaml}(r),function(e){var t=/(?:\\.|[^\\\n\r]|(?:\n|\r\n?)(?![\r\n]))/.source;function n(e){return e=e.replace(/<inner>/g,(function(){return t})),RegExp(/((?:^|[^\\])(?:\\{2})*)/.source+"(?:"+e+")")}var s=/(?:\\.|``(?:[^`\r\n]|`(?!`))+``|`[^`\r\n]+`|[^\\|\r\n`])+/.source,r=/\|?__(?:\|__)+\|?(?:(?:\n|\r\n?)|(?![\s\S]))/.source.replace(/__/g,(function(){return s})),o=/\|?[ \t]*:?-{3,}:?[ \t]*(?:\|[ \t]*:?-{3,}:?[ \t]*)+\|?(?:\n|\r\n?)/.source;e.languages.markdown=e.languages.extend("markup",{}),e.languages.insertBefore("markdown","prolog",{"front-matter-block":{pattern:/(^(?:\s*[\r\n])?)---(?!.)[\s\S]*?[\r\n]---(?!.)/,lookbehind:!0,greedy:!0,inside:{punctuation:/^---|---$/,"front-matter":{pattern:/\S+(?:\s+\S+)*/,alias:["yaml","language-yaml"],inside:e.languages.yaml}}},blockquote:{pattern:/^>(?:[\t ]*>)*/m,alias:"punctuation"},table:{pattern:RegExp("^"+r+o+"(?:"+r+")*","m"),inside:{"table-data-rows":{pattern:RegExp("^("+r+o+")(?:"+r+")*$"),lookbehind:!0,inside:{"table-data":{pattern:RegExp(s),inside:e.languages.markdown},punctuation:/\|/}},"table-line":{pattern:RegExp("^("+r+")"+o+"$"),lookbehind:!0,inside:{punctuation:/\||:?-{3,}:?/}},"table-header-row":{pattern:RegExp("^"+r+"$"),inside:{"table-header":{pattern:RegExp(s),alias:"important",inside:e.languages.markdown},punctuation:/\|/}}}},code:[{pattern:/((?:^|\n)[ \t]*\n|(?:^|\r\n?)[ \t]*\r\n?)(?: {4}|\t).+(?:(?:\n|\r\n?)(?: {4}|\t).+)*/,lookbehind:!0,alias:"keyword"},{pattern:/^```[\s\S]*?^```$/m,greedy:!0,inside:{"code-block":{pattern:/^(```.*(?:\n|\r\n?))[\s\S]+?(?=(?:\n|\r\n?)^```$)/m,lookbehind:!0},"code-language":{pattern:/^(```).+/,lookbehind:!0},punctuation:/```/}}],title:[{pattern:/\S.*(?:\n|\r\n?)(?:==+|--+)(?=[ \t]*$)/m,alias:"important",inside:{punctuation:/==+$|--+$/}},{pattern:/(^\s*)#.+/m,lookbehind:!0,alias:"important",inside:{punctuation:/^#+|#+$/}}],hr:{pattern:/(^\s*)([*-])(?:[\t ]*\2){2,}(?=\s*$)/m,lookbehind:!0,alias:"punctuation"},list:{pattern:/(^\s*)(?:[*+-]|\d+\.)(?=[\t ].)/m,lookbehind:!0,alias:"punctuation"},"url-reference":{pattern:/!?\[[^\]]+\]:[\t ]+(?:\S+|<(?:\\.|[^>\\])+>)(?:[\t ]+(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\)))?/,inside:{variable:{pattern:/^(!?\[)[^\]]+/,lookbehind:!0},string:/(?:"(?:\\.|[^"\\])*"|'(?:\\.|[^'\\])*'|\((?:\\.|[^)\\])*\))$/,punctuation:/^[\[\]!:]|[<>]/},alias:"url"},bold:{pattern:n(/\b__(?:(?!_)<inner>|_(?:(?!_)<inner>)+_)+__\b|\*\*(?:(?!\*)<inner>|\*(?:(?!\*)<inner>)+\*)+\*\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^..)[\s\S]+(?=..$)/,lookbehind:!0,inside:{}},punctuation:/\*\*|__/}},italic:{pattern:n(/\b_(?:(?!_)<inner>|__(?:(?!_)<inner>)+__)+_\b|\*(?:(?!\*)<inner>|\*\*(?:(?!\*)<inner>)+\*\*)+\*/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^.)[\s\S]+(?=.$)/,lookbehind:!0,inside:{}},punctuation:/[*_]/}},strike:{pattern:n(/(~~?)(?:(?!~)<inner>)+\2/.source),lookbehind:!0,greedy:!0,inside:{content:{pattern:/(^~~?)[\s\S]+(?=\1$)/,lookbehind:!0,inside:{}},punctuation:/~~?/}},"code-snippet":{pattern:/(^|[^\\`])(?:``[^`\r\n]+(?:`[^`\r\n]+)*``(?!`)|`[^`\r\n]+`(?!`))/,lookbehind:!0,greedy:!0,alias:["code","keyword"]},url:{pattern:n(/!?\[(?:(?!\])<inner>)+\](?:\([^\s)]+(?:[\t ]+"(?:\\.|[^"\\])*")?\)|[ \t]?\[(?:(?!\])<inner>)+\])/.source),lookbehind:!0,greedy:!0,inside:{operator:/^!/,content:{pattern:/(^\[)[^\]]+(?=\])/,lookbehind:!0,inside:{}},variable:{pattern:/(^\][ \t]?\[)[^\]]+(?=\]$)/,lookbehind:!0},url:{pattern:/(^\]\()[^\s)]+/,lookbehind:!0},string:{pattern:/(^[ \t]+)"(?:\\.|[^"\\])*"(?=\)$)/,lookbehind:!0}}}}),["url","bold","italic","strike"].forEach((function(t){["url","bold","italic","strike","code-snippet"].forEach((function(n){t!==n&&(e.languages.markdown[t].inside.content.inside[n]=e.languages.markdown[n])}))})),e.hooks.add("after-tokenize",(function(e){"markdown"!==e.language&&"md"!==e.language||function e(t){if(t&&"string"!=typeof t)for(var n=0,s=t.length;n<s;n++){var r=t[n];if("code"===r.type){var o=r.content[1],a=r.content[3];if(o&&a&&"code-language"===o.type&&"code-block"===a.type&&"string"==typeof o.content){var i=o.content.replace(/\b#/g,"sharp").replace(/\b\+\+/g,"pp"),c="language-"+(i=(/[a-z][\w-]*/i.exec(i)||[""])[0].toLowerCase());a.alias?"string"==typeof a.alias?a.alias=[a.alias,c]:a.alias.push(c):a.alias=[c]}}else e(r.content)}}(e.tokens)})),e.hooks.add("wrap",(function(t){if("code-block"===t.type){for(var n="",s=0,r=t.classes.length;s<r;s++){var o=t.classes[s],l=/language-(.+)/.exec(o);if(l){n=l[1];break}}var d,p=e.languages[n];if(p)t.content=e.highlight((d=t.content,d.replace(a,"").replace(/&(\w{1,8}|#x?[\da-f]{1,8});/gi,(function(e,t){var n;if("#"===(t=t.toLowerCase())[0])return n="x"===t[1]?parseInt(t.slice(2),16):Number(t.slice(1)),c(n);var s=i[t];return s||e}))),p,n);else if(n&&"none"!==n&&e.plugins.autoloader){var u="md-"+(new Date).valueOf()+"-"+Math.floor(1e16*Math.random());t.attributes.id=u,e.plugins.autoloader.loadLanguages(n,(function(){var t=document.getElementById(u);t&&(t.innerHTML=e.highlight(t.textContent,e.languages[n],n))}))}}}));var a=RegExp(e.languages.markup.tag.pattern.source,"gi"),i={amp:"&",lt:"<",gt:">",quot:'"'},c=String.fromCodePoint||String.fromCharCode;e.languages.md=e.languages.markdown}(r),r.languages.graphql={comment:/#.*/,description:{pattern:/(?:"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*")(?=\s*[a-z_])/i,greedy:!0,alias:"string",inside:{"language-markdown":{pattern:/(^"(?:"")?)(?!\1)[\s\S]+(?=\1$)/,lookbehind:!0,inside:r.languages.markdown}}},string:{pattern:/"""(?:[^"]|(?!""")")*"""|"(?:\\.|[^\\"\r\n])*"/,greedy:!0},number:/(?:\B-|\b)\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,boolean:/\b(?:false|true)\b/,variable:/\$[a-z_]\w*/i,directive:{pattern:/@[a-z_]\w*/i,alias:"function"},"attr-name":{pattern:/\b[a-z_]\w*(?=\s*(?:\((?:[^()"]|"(?:\\.|[^\\"\r\n])*")*\))?:)/i,greedy:!0},"atom-input":{pattern:/\b[A-Z]\w*Input\b/,alias:"class-name"},scalar:/\b(?:Boolean|Float|ID|Int|String)\b/,constant:/\b[A-Z][A-Z_\d]*\b/,"class-name":{pattern:/(\b(?:enum|implements|interface|on|scalar|type|union)\s+|&\s*|:\s*|\[)[A-Z_]\w*/,lookbehind:!0},fragment:{pattern:/(\bfragment\s+|\.{3}\s*(?!on\b))[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-mutation":{pattern:/(\bmutation\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},"definition-query":{pattern:/(\bquery\s+)[a-zA-Z_]\w*/,lookbehind:!0,alias:"function"},keyword:/\b(?:directive|enum|extend|fragment|implements|input|interface|mutation|on|query|repeatable|scalar|schema|subscription|type|union)\b/,operator:/[!=|&]|\.{3}/,"property-query":/\w+(?=\s*\()/,object:/\w+(?=\s*\{)/,punctuation:/[!(){}\[\]:=,]/,property:/\w+/},r.hooks.add("after-tokenize",(function(e){if("graphql"===e.language)for(var t=e.tokens.filter((function(e){return"string"!=typeof e&&"comment"!==e.type&&"scalar"!==e.type})),n=0;n<t.length;){var s=t[n++];if("keyword"===s.type&&"mutation"===s.content){var r=[];if(p(["definition-mutation","punctuation"])&&"("===d(1).content){n+=2;var o=u(/^\($/,/^\)$/);if(-1===o)continue;for(;n<o;n++){var a=d(0);"variable"===a.type&&(m(a,"variable-input"),r.push(a.content))}n=o+1}if(p(["punctuation","property-query"])&&"{"===d(0).content&&(n++,m(d(0),"property-mutation"),r.length>0)){var i=u(/^\{$/,/^\}$/);if(-1===i)continue;for(var c=n;c<i;c++){var l=t[c];"variable"===l.type&&r.indexOf(l.content)>=0&&m(l,"variable-input")}}}}function d(e){return t[n+e]}function p(e,t){t=t||0;for(var n=0;n<e.length;n++){var s=d(n+t);if(!s||s.type!==e[n])return!1}return!0}function u(e,s){for(var r=1,o=n;o<t.length;o++){var a=t[o],i=a.content;if("punctuation"===a.type&&"string"==typeof i)if(e.test(i))r++;else if(s.test(i)&&0===--r)return o}return-1}function m(e,t){var n=e.alias;n?Array.isArray(n)||(e.alias=n=[n]):e.alias=n=[],n.push(t)}})),r.languages.sql={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|(?:--|\/\/|#).*)/,lookbehind:!0},variable:[{pattern:/@(["'`])(?:\\[\s\S]|(?!\1)[^\\])+\1/,greedy:!0},/@[\w.$]+/],string:{pattern:/(^|[^@\\])("|')(?:\\[\s\S]|(?!\2)[^\\]|\2\2)*\2/,greedy:!0,lookbehind:!0},identifier:{pattern:/(^|[^@\\])`(?:\\[\s\S]|[^`\\]|``)*`/,greedy:!0,lookbehind:!0,inside:{punctuation:/^`|`$/}},function:/\b(?:AVG|COUNT|FIRST|FORMAT|LAST|LCASE|LEN|MAX|MID|MIN|MOD|NOW|ROUND|SUM|UCASE)(?=\s*\()/i,keyword:/\b(?:ACTION|ADD|AFTER|ALGORITHM|ALL|ALTER|ANALYZE|ANY|APPLY|AS|ASC|AUTHORIZATION|AUTO_INCREMENT|BACKUP|BDB|BEGIN|BERKELEYDB|BIGINT|BINARY|BIT|BLOB|BOOL|BOOLEAN|BREAK|BROWSE|BTREE|BULK|BY|CALL|CASCADED?|CASE|CHAIN|CHAR(?:ACTER|SET)?|CHECK(?:POINT)?|CLOSE|CLUSTERED|COALESCE|COLLATE|COLUMNS?|COMMENT|COMMIT(?:TED)?|COMPUTE|CONNECT|CONSISTENT|CONSTRAINT|CONTAINS(?:TABLE)?|CONTINUE|CONVERT|CREATE|CROSS|CURRENT(?:_DATE|_TIME|_TIMESTAMP|_USER)?|CURSOR|CYCLE|DATA(?:BASES?)?|DATE(?:TIME)?|DAY|DBCC|DEALLOCATE|DEC|DECIMAL|DECLARE|DEFAULT|DEFINER|DELAYED|DELETE|DELIMITERS?|DENY|DESC|DESCRIBE|DETERMINISTIC|DISABLE|DISCARD|DISK|DISTINCT|DISTINCTROW|DISTRIBUTED|DO|DOUBLE|DROP|DUMMY|DUMP(?:FILE)?|DUPLICATE|ELSE(?:IF)?|ENABLE|ENCLOSED|END|ENGINE|ENUM|ERRLVL|ERRORS|ESCAPED?|EXCEPT|EXEC(?:UTE)?|EXISTS|EXIT|EXPLAIN|EXTENDED|FETCH|FIELDS|FILE|FILLFACTOR|FIRST|FIXED|FLOAT|FOLLOWING|FOR(?: EACH ROW)?|FORCE|FOREIGN|FREETEXT(?:TABLE)?|FROM|FULL|FUNCTION|GEOMETRY(?:COLLECTION)?|GLOBAL|GOTO|GRANT|GROUP|HANDLER|HASH|HAVING|HOLDLOCK|HOUR|IDENTITY(?:COL|_INSERT)?|IF|IGNORE|IMPORT|INDEX|INFILE|INNER|INNODB|INOUT|INSERT|INT|INTEGER|INTERSECT|INTERVAL|INTO|INVOKER|ISOLATION|ITERATE|JOIN|KEYS?|KILL|LANGUAGE|LAST|LEAVE|LEFT|LEVEL|LIMIT|LINENO|LINES|LINESTRING|LOAD|LOCAL|LOCK|LONG(?:BLOB|TEXT)|LOOP|MATCH(?:ED)?|MEDIUM(?:BLOB|INT|TEXT)|MERGE|MIDDLEINT|MINUTE|MODE|MODIFIES|MODIFY|MONTH|MULTI(?:LINESTRING|POINT|POLYGON)|NATIONAL|NATURAL|NCHAR|NEXT|NO|NONCLUSTERED|NULLIF|NUMERIC|OFF?|OFFSETS?|ON|OPEN(?:DATASOURCE|QUERY|ROWSET)?|OPTIMIZE|OPTION(?:ALLY)?|ORDER|OUT(?:ER|FILE)?|OVER|PARTIAL|PARTITION|PERCENT|PIVOT|PLAN|POINT|POLYGON|PRECEDING|PRECISION|PREPARE|PREV|PRIMARY|PRINT|PRIVILEGES|PROC(?:EDURE)?|PUBLIC|PURGE|QUICK|RAISERROR|READS?|REAL|RECONFIGURE|REFERENCES|RELEASE|RENAME|REPEAT(?:ABLE)?|REPLACE|REPLICATION|REQUIRE|RESIGNAL|RESTORE|RESTRICT|RETURN(?:ING|S)?|REVOKE|RIGHT|ROLLBACK|ROUTINE|ROW(?:COUNT|GUIDCOL|S)?|RTREE|RULE|SAVE(?:POINT)?|SCHEMA|SECOND|SELECT|SERIAL(?:IZABLE)?|SESSION(?:_USER)?|SET(?:USER)?|SHARE|SHOW|SHUTDOWN|SIMPLE|SMALLINT|SNAPSHOT|SOME|SONAME|SQL|START(?:ING)?|STATISTICS|STATUS|STRIPED|SYSTEM_USER|TABLES?|TABLESPACE|TEMP(?:ORARY|TABLE)?|TERMINATED|TEXT(?:SIZE)?|THEN|TIME(?:STAMP)?|TINY(?:BLOB|INT|TEXT)|TOP?|TRAN(?:SACTIONS?)?|TRIGGER|TRUNCATE|TSEQUAL|TYPES?|UNBOUNDED|UNCOMMITTED|UNDEFINED|UNION|UNIQUE|UNLOCK|UNPIVOT|UNSIGNED|UPDATE(?:TEXT)?|USAGE|USE|USER|USING|VALUES?|VAR(?:BINARY|CHAR|CHARACTER|YING)|VIEW|WAITFOR|WARNINGS|WHEN|WHERE|WHILE|WITH(?: ROLLUP|IN)?|WORK|WRITE(?:TEXT)?|YEAR)\b/i,boolean:/\b(?:FALSE|NULL|TRUE)\b/i,number:/\b0x[\da-f]+\b|\b\d+(?:\.\d*)?|\B\.\d+\b/i,operator:/[-+*\/=%^~]|&&?|\|\|?|!=?|<(?:=>?|<|>)?|>[>=]?|\b(?:AND|BETWEEN|DIV|ILIKE|IN|IS|LIKE|NOT|OR|REGEXP|RLIKE|SOUNDS LIKE|XOR)\b/i,punctuation:/[;[\]()`,.]/},function(e){var t=e.languages.javascript["template-string"],n=t.pattern.source,s=t.inside.interpolation,r=s.inside["interpolation-punctuation"],o=s.pattern.source;function a(t,s){if(e.languages[t])return{pattern:RegExp("((?:"+s+")\\s*)"+n),lookbehind:!0,greedy:!0,inside:{"template-punctuation":{pattern:/^`|`$/,alias:"string"},"embedded-code":{pattern:/[\s\S]+/,alias:t}}}}function i(e,t){return"___"+t.toUpperCase()+"_"+e+"___"}function c(t,n,s){var r={code:t,grammar:n,language:s};return e.hooks.run("before-tokenize",r),r.tokens=e.tokenize(r.code,r.grammar),e.hooks.run("after-tokenize",r),r.tokens}function l(t){var n={};n["interpolation-punctuation"]=r;var o=e.tokenize(t,n);if(3===o.length){var a=[1,1];a.push.apply(a,c(o[1],e.languages.javascript,"javascript")),o.splice.apply(o,a)}return new e.Token("interpolation",o,s.alias,t)}function d(t,n,s){var r=e.tokenize(t,{interpolation:{pattern:RegExp(o),lookbehind:!0}}),a=0,d={},p=c(r.map((function(e){if("string"==typeof e)return e;for(var n,r=e.content;-1!==t.indexOf(n=i(a++,s)););return d[n]=r,n})).join(""),n,s),u=Object.keys(d);return a=0,function e(t){for(var n=0;n<t.length;n++){if(a>=u.length)return;var s=t[n];if("string"==typeof s||"string"==typeof s.content){var r=u[a],o="string"==typeof s?s:s.content,i=o.indexOf(r);if(-1!==i){++a;var c=o.substring(0,i),p=l(d[r]),m=o.substring(i+r.length),h=[];if(c&&h.push(c),h.push(p),m){var g=[m];e(g),h.push.apply(h,g)}"string"==typeof s?(t.splice.apply(t,[n,1].concat(h)),n+=h.length-1):s.content=h}}else{var f=s.content;Array.isArray(f)?e(f):e([f])}}}(p),new e.Token(s,p,"language-"+s,t)}e.languages.javascript["template-string"]=[a("css",/\b(?:styled(?:\([^)]*\))?(?:\s*\.\s*\w+(?:\([^)]*\))*)*|css(?:\s*\.\s*(?:global|resolve))?|createGlobalStyle|keyframes)/.source),a("html",/\bhtml|\.\s*(?:inner|outer)HTML\s*\+?=/.source),a("svg",/\bsvg/.source),a("markdown",/\b(?:markdown|md)/.source),a("graphql",/\b(?:gql|graphql(?:\s*\.\s*experimental)?)/.source),a("sql",/\bsql/.source),t].filter(Boolean);var p={javascript:!0,js:!0,typescript:!0,ts:!0,jsx:!0,tsx:!0};function u(e){return"string"==typeof e?e:Array.isArray(e)?e.map(u).join(""):u(e.content)}e.hooks.add("after-tokenize",(function(t){t.language in p&&function t(n){for(var s=0,r=n.length;s<r;s++){var o=n[s];if("string"!=typeof o){var a=o.content;if(Array.isArray(a))if("template-string"===o.type){var i=a[1];if(3===a.length&&"string"!=typeof i&&"embedded-code"===i.type){var c=u(i),l=i.alias,p=Array.isArray(l)?l[0]:l,m=e.languages[p];if(!m)continue;a[1]=d(c,m,p)}}else t(a);else"string"!=typeof a&&t([a])}}}(t.tokens)}))}(r),function(e){e.languages.typescript=e.languages.extend("javascript",{"class-name":{pattern:/(\b(?:class|extends|implements|instanceof|interface|new|type)\s+)(?!keyof\b)(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*(?:\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>)?/,lookbehind:!0,greedy:!0,inside:null},builtin:/\b(?:Array|Function|Promise|any|boolean|console|never|number|string|symbol|unknown)\b/}),e.languages.typescript.keyword.push(/\b(?:abstract|declare|is|keyof|readonly|require)\b/,/\b(?:asserts|infer|interface|module|namespace|type)\b(?=\s*(?:[{_$a-zA-Z\xA0-\uFFFF]|$))/,/\btype\b(?=\s*(?:[\{*]|$))/),delete e.languages.typescript.parameter,delete e.languages.typescript["literal-property"];var t=e.languages.extend("typescript",{});delete t["class-name"],e.languages.typescript["class-name"].inside=t,e.languages.insertBefore("typescript","function",{decorator:{pattern:/@[$\w\xA0-\uFFFF]+/,inside:{at:{pattern:/^@/,alias:"operator"},function:/^[\s\S]+/}},"generic-function":{pattern:/#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*\s*<(?:[^<>]|<(?:[^<>]|<[^<>]*>)*>)*>(?=\s*\()/,greedy:!0,inside:{function:/^#?(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/,generic:{pattern:/<[\s\S]+/,alias:"class-name",inside:t}}}}),e.languages.ts=e.languages.typescript}(r),function(e){function t(e,t){return RegExp(e.replace(/<ID>/g,(function(){return/(?!\s)[_$a-zA-Z\xA0-\uFFFF](?:(?!\s)[$\w\xA0-\uFFFF])*/.source})),t)}e.languages.insertBefore("javascript","function-variable",{"method-variable":{pattern:RegExp("(\\.\\s*)"+e.languages.javascript["function-variable"].pattern.source),lookbehind:!0,alias:["function-variable","method","function","property-access"]}}),e.languages.insertBefore("javascript","function",{method:{pattern:RegExp("(\\.\\s*)"+e.languages.javascript.function.source),lookbehind:!0,alias:["function","property-access"]}}),e.languages.insertBefore("javascript","constant",{"known-class-name":[{pattern:/\b(?:(?:Float(?:32|64)|(?:Int|Uint)(?:8|16|32)|Uint8Clamped)?Array|ArrayBuffer|BigInt|Boolean|DataView|Date|Error|Function|Intl|JSON|(?:Weak)?(?:Map|Set)|Math|Number|Object|Promise|Proxy|Reflect|RegExp|String|Symbol|WebAssembly)\b/,alias:"class-name"},{pattern:/\b(?:[A-Z]\w*)Error\b/,alias:"class-name"}]}),e.languages.insertBefore("javascript","keyword",{imports:{pattern:t(/(\bimport\b\s*)(?:<ID>(?:\s*,\s*(?:\*\s*as\s+<ID>|\{[^{}]*\}))?|\*\s*as\s+<ID>|\{[^{}]*\})(?=\s*\bfrom\b)/.source),lookbehind:!0,inside:e.languages.javascript},exports:{pattern:t(/(\bexport\b\s*)(?:\*(?:\s*as\s+<ID>)?(?=\s*\bfrom\b)|\{[^{}]*\})/.source),lookbehind:!0,inside:e.languages.javascript}}),e.languages.javascript.keyword.unshift({pattern:/\b(?:as|default|export|from|import)\b/,alias:"module"},{pattern:/\b(?:await|break|catch|continue|do|else|finally|for|if|return|switch|throw|try|while|yield)\b/,alias:"control-flow"},{pattern:/\bnull\b/,alias:["null","nil"]},{pattern:/\bundefined\b/,alias:"nil"}),e.languages.insertBefore("javascript","operator",{spread:{pattern:/\.{3}/,alias:"operator"},arrow:{pattern:/=>/,alias:"operator"}}),e.languages.insertBefore("javascript","punctuation",{"property-access":{pattern:t(/(\.\s*)#?<ID>/.source),lookbehind:!0},"maybe-class-name":{pattern:/(^|[^$\w\xA0-\uFFFF])[A-Z][$\w\xA0-\uFFFF]+/,lookbehind:!0},dom:{pattern:/\b(?:document|(?:local|session)Storage|location|navigator|performance|window)\b/,alias:"variable"},console:{pattern:/\bconsole(?=\s*\.)/,alias:"class-name"}});for(var n=["function","function-variable","method","method-variable","property-access"],s=0;s<n.length;s++){var r=n[s],o=e.languages.javascript[r];"RegExp"===e.util.type(o)&&(o=e.languages.javascript[r]={pattern:o});var a=o.inside||{};o.inside=a,a["maybe-class-name"]=/^[A-Z][\s\S]*/}}(r),function(e){var t=e.util.clone(e.languages.javascript),n=/(?:\s|\/\/.*(?!.)|\/\*(?:[^*]|\*(?!\/))\*\/)/.source,s=/(?:\{(?:\{(?:\{[^{}]*\}|[^{}])*\}|[^{}])*\})/.source,r=/(?:\{<S>*\.{3}(?:[^{}]|<BRACES>)*\})/.source;function o(e,t){return e=e.replace(/<S>/g,(function(){return n})).replace(/<BRACES>/g,(function(){return s})).replace(/<SPREAD>/g,(function(){return r})),RegExp(e,t)}r=o(r).source,e.languages.jsx=e.languages.extend("markup",t),e.languages.jsx.tag.pattern=o(/<\/?(?:[\w.:-]+(?:<S>+(?:[\w.:$-]+(?:=(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s{'"/>=]+|<BRACES>))?|<SPREAD>))*<S>*\/?)?>/.source),e.languages.jsx.tag.inside.tag.pattern=/^<\/?[^\s>\/]*/,e.languages.jsx.tag.inside["attr-value"].pattern=/=(?!\{)(?:"(?:\\[\s\S]|[^\\"])*"|'(?:\\[\s\S]|[^\\'])*'|[^\s'">]+)/,e.languages.jsx.tag.inside.tag.inside["class-name"]=/^[A-Z]\w*(?:\.[A-Z]\w*)*$/,e.languages.jsx.tag.inside.comment=t.comment,e.languages.insertBefore("inside","attr-name",{spread:{pattern:o(/<SPREAD>/.source),inside:e.languages.jsx}},e.languages.jsx.tag),e.languages.insertBefore("inside","special-attr",{script:{pattern:o(/=<BRACES>/.source),alias:"language-javascript",inside:{"script-punctuation":{pattern:/^=(?=\{)/,alias:"punctuation"},rest:e.languages.jsx}}},e.languages.jsx.tag);var a=function(e){return e?"string"==typeof e?e:"string"==typeof e.content?e.content:e.content.map(a).join(""):""},i=function(t){for(var n=[],s=0;s<t.length;s++){var r=t[s],o=!1;if("string"!=typeof r&&("tag"===r.type&&r.content[0]&&"tag"===r.content[0].type?"</"===r.content[0].content[0].content?n.length>0&&n[n.length-1].tagName===a(r.content[0].content[1])&&n.pop():"/>"===r.content[r.content.length-1].content||n.push({tagName:a(r.content[0].content[1]),openedBraces:0}):n.length>0&&"punctuation"===r.type&&"{"===r.content?n[n.length-1].openedBraces++:n.length>0&&n[n.length-1].openedBraces>0&&"punctuation"===r.type&&"}"===r.content?n[n.length-1].openedBraces--:o=!0),(o||"string"==typeof r)&&n.length>0&&0===n[n.length-1].openedBraces){var c=a(r);s<t.length-1&&("string"==typeof t[s+1]||"plain-text"===t[s+1].type)&&(c+=a(t[s+1]),t.splice(s+1,1)),s>0&&("string"==typeof t[s-1]||"plain-text"===t[s-1].type)&&(c=a(t[s-1])+c,t.splice(s-1,1),s--),t[s]=new e.Token("plain-text",c,null,c)}r.content&&"string"!=typeof r.content&&i(r.content)}};e.hooks.add("after-tokenize",(function(e){"jsx"!==e.language&&"tsx"!==e.language||i(e.tokens)}))}(r),function(e){e.languages.diff={coord:[/^(?:\*{3}|-{3}|\+{3}).*$/m,/^@@.*@@$/m,/^\d.*$/m]};var t={"deleted-sign":"-","deleted-arrow":"<","inserted-sign":"+","inserted-arrow":">",unchanged:" ",diff:"!"};Object.keys(t).forEach((function(n){var s=t[n],r=[];/^\w+$/.test(n)||r.push(/\w+/.exec(n)[0]),"diff"===n&&r.push("bold"),e.languages.diff[n]={pattern:RegExp("^(?:["+s+"].*(?:\r\n?|\n|(?![\\s\\S])))+","m"),alias:r,inside:{line:{pattern:/(.)(?=[\s\S]).*(?:\r\n?|\n)?/,lookbehind:!0},prefix:{pattern:/[\s\S]/,alias:/\w+/.exec(n)[0]}}}})),Object.defineProperty(e.languages.diff,"PREFIXES",{value:t})}(r),r.languages.git={comment:/^#.*/m,deleted:/^[-\u2013].*/m,inserted:/^\+.*/m,string:/("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,command:{pattern:/^.*\$ git .*$/m,inside:{parameter:/\s--?\w+/}},coord:/^@@.*@@$/m,"commit-sha1":/^commit \w{40}$/m},r.languages.go=r.languages.extend("clike",{string:{pattern:/(^|[^\\])"(?:\\.|[^"\\\r\n])*"|`[^`]*`/,lookbehind:!0,greedy:!0},keyword:/\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,boolean:/\b(?:_|false|iota|nil|true)\b/,number:[/\b0(?:b[01_]+|o[0-7_]+)i?\b/i,/\b0x(?:[a-f\d_]+(?:\.[a-f\d_]*)?|\.[a-f\d_]+)(?:p[+-]?\d+(?:_\d+)*)?i?(?!\w)/i,/(?:\b\d[\d_]*(?:\.[\d_]*)?|\B\.\d[\d_]*)(?:e[+-]?[\d_]+)?i?(?!\w)/i],operator:/[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,builtin:/\b(?:append|bool|byte|cap|close|complex|complex(?:64|128)|copy|delete|error|float(?:32|64)|u?int(?:8|16|32|64)?|imag|len|make|new|panic|print(?:ln)?|real|recover|rune|string|uintptr)\b/}),r.languages.insertBefore("go","string",{char:{pattern:/'(?:\\.|[^'\\\r\n]){0,10}'/,greedy:!0}}),delete r.languages.go["class-name"],function(e){function t(e,t){return"___"+e.toUpperCase()+t+"___"}Object.defineProperties(e.languages["markup-templating"]={},{buildPlaceholders:{value:function(n,s,r,o){if(n.language===s){var a=n.tokenStack=[];n.code=n.code.replace(r,(function(e){if("function"==typeof o&&!o(e))return e;for(var r,i=a.length;-1!==n.code.indexOf(r=t(s,i));)++i;return a[i]=e,r})),n.grammar=e.languages.markup}}},tokenizePlaceholders:{value:function(n,s){if(n.language===s&&n.tokenStack){n.grammar=e.languages[s];var r=0,o=Object.keys(n.tokenStack);!function a(i){for(var c=0;c<i.length&&!(r>=o.length);c++){var l=i[c];if("string"==typeof l||l.content&&"string"==typeof l.content){var d=o[r],p=n.tokenStack[d],u="string"==typeof l?l:l.content,m=t(s,d),h=u.indexOf(m);if(h>-1){++r;var g=u.substring(0,h),f=new e.Token(s,e.tokenize(p,n.grammar),"language-"+s,p),b=u.substring(h+m.length),y=[];g&&y.push.apply(y,a([g])),y.push(f),b&&y.push.apply(y,a([b])),"string"==typeof l?i.splice.apply(i,[c,1].concat(y)):l.content=y}}else l.content&&a(l.content)}return i}(n.tokens)}}}})}(r),function(e){e.languages.handlebars={comment:/\{\{![\s\S]*?\}\}/,delimiter:{pattern:/^\{\{\{?|\}\}\}?$/,alias:"punctuation"},string:/(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,number:/\b0x[\dA-Fa-f]+\b|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][+-]?\d+)?/,boolean:/\b(?:false|true)\b/,block:{pattern:/^(\s*(?:~\s*)?)[#\/]\S+?(?=\s*(?:~\s*)?$|\s)/,lookbehind:!0,alias:"keyword"},brackets:{pattern:/\[[^\]]+\]/,inside:{punctuation:/\[|\]/,variable:/[\s\S]+/}},punctuation:/[!"#%&':()*+,.\/;<=>@\[\\\]^`{|}~]/,variable:/[^!"#%&'()*+,\/;<=>@\[\\\]^`{|}~\s]+/},e.hooks.add("before-tokenize",(function(t){e.languages["markup-templating"].buildPlaceholders(t,"handlebars",/\{\{\{[\s\S]+?\}\}\}|\{\{[\s\S]+?\}\}/g)})),e.hooks.add("after-tokenize",(function(t){e.languages["markup-templating"].tokenizePlaceholders(t,"handlebars")})),e.languages.hbs=e.languages.handlebars}(r),r.languages.json={property:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?=\s*:)/,lookbehind:!0,greedy:!0},string:{pattern:/(^|[^\\])"(?:\\.|[^\\"\r\n])*"(?!\s*:)/,lookbehind:!0,greedy:!0},comment:{pattern:/\/\/.*|\/\*[\s\S]*?(?:\*\/|$)/,greedy:!0},number:/-?\b\d+(?:\.\d+)?(?:e[+-]?\d+)?\b/i,punctuation:/[{}[\],]/,operator:/:/,boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"}},r.languages.webmanifest=r.languages.json,r.languages.less=r.languages.extend("css",{comment:[/\/\*[\s\S]*?\*\//,{pattern:/(^|[^\\])\/\/.*/,lookbehind:!0}],atrule:{pattern:/@[\w-](?:\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{punctuation:/[:()]/}},selector:{pattern:/(?:@\{[\w-]+\}|[^{};\s@])(?:@\{[\w-]+\}|\((?:[^(){}]|\([^(){}]*\))*\)|[^(){};@\s]|\s+(?!\s))*?(?=\s*\{)/,inside:{variable:/@+[\w-]+/}},property:/(?:@\{[\w-]+\}|[\w-])+(?:\+_?)?(?=\s*:)/,operator:/[+\-*\/]/}),r.languages.insertBefore("less","property",{variable:[{pattern:/@[\w-]+\s*:/,inside:{punctuation:/:/}},/@@?[\w-]+/],"mixin-usage":{pattern:/([{;]\s*)[.#](?!\d)[\w-].*?(?=[(;])/,lookbehind:!0,alias:"function"}}),r.languages.makefile={comment:{pattern:/(^|[^\\])#(?:\\(?:\r\n|[\s\S])|[^\\\r\n])*/,lookbehind:!0},string:{pattern:/(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,greedy:!0},"builtin-target":{pattern:/\.[A-Z][^:#=\s]+(?=\s*:(?!=))/,alias:"builtin"},target:{pattern:/^(?:[^:=\s]|[ \t]+(?![\s:]))+(?=\s*:(?!=))/m,alias:"symbol",inside:{variable:/\$+(?:(?!\$)[^(){}:#=\s]+|(?=[({]))/}},variable:/\$+(?:(?!\$)[^(){}:#=\s]+|\([@*%<^+?][DF]\)|(?=[({]))/,keyword:/-include\b|\b(?:define|else|endef|endif|export|ifn?def|ifn?eq|include|override|private|sinclude|undefine|unexport|vpath)\b/,function:{pattern:/(\()(?:abspath|addsuffix|and|basename|call|dir|error|eval|file|filter(?:-out)?|findstring|firstword|flavor|foreach|guile|if|info|join|lastword|load|notdir|or|origin|patsubst|realpath|shell|sort|strip|subst|suffix|value|warning|wildcard|word(?:list|s)?)(?=[ \t])/,lookbehind:!0},operator:/(?:::|[?:+!])?=|[|@]/,punctuation:/[:;(){}]/},r.languages.objectivec=r.languages.extend("c",{string:{pattern:/@?"(?:\\(?:\r\n|[\s\S])|[^"\\\r\n])*"/,greedy:!0},keyword:/\b(?:asm|auto|break|case|char|const|continue|default|do|double|else|enum|extern|float|for|goto|if|in|inline|int|long|register|return|self|short|signed|sizeof|static|struct|super|switch|typedef|typeof|union|unsigned|void|volatile|while)\b|(?:@interface|@end|@implementation|@protocol|@class|@public|@protected|@private|@property|@try|@catch|@finally|@throw|@synthesize|@dynamic|@selector)\b/,operator:/-[->]?|\+\+?|!=?|<<?=?|>>?=?|==?|&&?|\|\|?|[~^%?*\/@]/}),delete r.languages.objectivec["class-name"],r.languages.objc=r.languages.objectivec,r.languages.ocaml={comment:{pattern:/\(\*[\s\S]*?\*\)/,greedy:!0},char:{pattern:/'(?:[^\\\r\n']|\\(?:.|[ox]?[0-9a-f]{1,3}))'/i,greedy:!0},string:[{pattern:/"(?:\\(?:[\s\S]|\r\n)|[^\\\r\n"])*"/,greedy:!0},{pattern:/\{([a-z_]*)\|[\s\S]*?\|\1\}/,greedy:!0}],number:[/\b(?:0b[01][01_]*|0o[0-7][0-7_]*)\b/i,/\b0x[a-f0-9][a-f0-9_]*(?:\.[a-f0-9_]*)?(?:p[+-]?\d[\d_]*)?(?!\w)/i,/\b\d[\d_]*(?:\.[\d_]*)?(?:e[+-]?\d[\d_]*)?(?!\w)/i],directive:{pattern:/\B#\w+/,alias:"property"},label:{pattern:/\B~\w+/,alias:"property"},"type-variable":{pattern:/\B'\w+/,alias:"function"},variant:{pattern:/`\w+/,alias:"symbol"},keyword:/\b(?:as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|match|method|module|mutable|new|nonrec|object|of|open|private|rec|sig|struct|then|to|try|type|val|value|virtual|when|where|while|with)\b/,boolean:/\b(?:false|true)\b/,"operator-like-punctuation":{pattern:/\[[<>|]|[>|]\]|\{<|>\}/,alias:"punctuation"},operator:/\.[.~]|:[=>]|[=<>@^|&+\-*\/$%!?~][!$%&*+\-.\/:<=>?@^|~]*|\b(?:and|asr|land|lor|lsl|lsr|lxor|mod|or)\b/,punctuation:/;;|::|[(){}\[\].,:;#]|\b_\b/},r.languages.python={comment:{pattern:/(^|[^\\])#.*/,lookbehind:!0,greedy:!0},"string-interpolation":{pattern:/(?:f|fr|rf)(?:("""|''')[\s\S]*?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2)/i,greedy:!0,inside:{interpolation:{pattern:/((?:^|[^{])(?:\{\{)*)\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}]|\{(?!\{)(?:[^{}])+\})+\})+\}/,lookbehind:!0,inside:{"format-spec":{pattern:/(:)[^:(){}]+(?=\}$)/,lookbehind:!0},"conversion-option":{pattern:/![sra](?=[:}]$)/,alias:"punctuation"},rest:null}},string:/[\s\S]+/}},"triple-quoted-string":{pattern:/(?:[rub]|br|rb)?("""|''')[\s\S]*?\1/i,greedy:!0,alias:"string"},string:{pattern:/(?:[rub]|br|rb)?("|')(?:\\.|(?!\1)[^\\\r\n])*\1/i,greedy:!0},function:{pattern:/((?:^|\s)def[ \t]+)[a-zA-Z_]\w*(?=\s*\()/g,lookbehind:!0},"class-name":{pattern:/(\bclass\s+)\w+/i,lookbehind:!0},decorator:{pattern:/(^[\t ]*)@\w+(?:\.\w+)*/m,lookbehind:!0,alias:["annotation","punctuation"],inside:{punctuation:/\./}},keyword:/\b(?:_(?=\s*:)|and|as|assert|async|await|break|case|class|continue|def|del|elif|else|except|exec|finally|for|from|global|if|import|in|is|lambda|match|nonlocal|not|or|pass|print|raise|return|try|while|with|yield)\b/,builtin:/\b(?:__import__|abs|all|any|apply|ascii|basestring|bin|bool|buffer|bytearray|bytes|callable|chr|classmethod|cmp|coerce|compile|complex|delattr|dict|dir|divmod|enumerate|eval|execfile|file|filter|float|format|frozenset|getattr|globals|hasattr|hash|help|hex|id|input|int|intern|isinstance|issubclass|iter|len|list|locals|long|map|max|memoryview|min|next|object|oct|open|ord|pow|property|range|raw_input|reduce|reload|repr|reversed|round|set|setattr|slice|sorted|staticmethod|str|sum|super|tuple|type|unichr|unicode|vars|xrange|zip)\b/,boolean:/\b(?:False|None|True)\b/,number:/\b0(?:b(?:_?[01])+|o(?:_?[0-7])+|x(?:_?[a-f0-9])+)\b|(?:\b\d+(?:_\d+)*(?:\.(?:\d+(?:_\d+)*)?)?|\B\.\d+(?:_\d+)*)(?:e[+-]?\d+(?:_\d+)*)?j?(?!\w)/i,operator:/[-+%=]=?|!=|:=|\*\*?=?|\/\/?=?|<[<=>]?|>[=>]?|[&|^~]/,punctuation:/[{}[\];(),.:]/},r.languages.python["string-interpolation"].inside.interpolation.inside.rest=r.languages.python,r.languages.py=r.languages.python,r.languages.reason=r.languages.extend("clike",{string:{pattern:/"(?:\\(?:\r\n|[\s\S])|[^\\\r\n"])*"/,greedy:!0},"class-name":/\b[A-Z]\w*/,keyword:/\b(?:and|as|assert|begin|class|constraint|do|done|downto|else|end|exception|external|for|fun|function|functor|if|in|include|inherit|initializer|lazy|let|method|module|mutable|new|nonrec|object|of|open|or|private|rec|sig|struct|switch|then|to|try|type|val|virtual|when|while|with)\b/,operator:/\.{3}|:[:=]|\|>|->|=(?:==?|>)?|<=?|>=?|[|^?'#!~`]|[+\-*\/]\.?|\b(?:asr|land|lor|lsl|lsr|lxor|mod)\b/}),r.languages.insertBefore("reason","class-name",{char:{pattern:/'(?:\\x[\da-f]{2}|\\o[0-3][0-7][0-7]|\\\d{3}|\\.|[^'\\\r\n])'/,greedy:!0},constructor:/\b[A-Z]\w*\b(?!\s*\.)/,label:{pattern:/\b[a-z]\w*(?=::)/,alias:"symbol"}}),delete r.languages.reason.function,function(e){e.languages.sass=e.languages.extend("css",{comment:{pattern:/^([ \t]*)\/[\/*].*(?:(?:\r?\n|\r)\1[ \t].+)*/m,lookbehind:!0,greedy:!0}}),e.languages.insertBefore("sass","atrule",{"atrule-line":{pattern:/^(?:[ \t]*)[@+=].+/m,greedy:!0,inside:{atrule:/(?:@[\w-]+|[+=])/}}}),delete e.languages.sass.atrule;var t=/\$[-\w]+|#\{\$[-\w]+\}/,n=[/[+*\/%]|[=!]=|<=?|>=?|\b(?:and|not|or)\b/,{pattern:/(\s)-(?=\s)/,lookbehind:!0}];e.languages.insertBefore("sass","property",{"variable-line":{pattern:/^[ \t]*\$.+/m,greedy:!0,inside:{punctuation:/:/,variable:t,operator:n}},"property-line":{pattern:/^[ \t]*(?:[^:\s]+ *:.*|:[^:\s].*)/m,greedy:!0,inside:{property:[/[^:\s]+(?=\s*:)/,{pattern:/(:)[^:\s]+/,lookbehind:!0}],punctuation:/:/,variable:t,operator:n,important:e.languages.sass.important}}}),delete e.languages.sass.property,delete e.languages.sass.important,e.languages.insertBefore("sass","punctuation",{selector:{pattern:/^([ \t]*)\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*(?:,(?:\r?\n|\r)\1[ \t]+\S(?:,[^,\r\n]+|[^,\r\n]*)(?:,[^,\r\n]+)*)*/m,lookbehind:!0,greedy:!0}})}(r),r.languages.scss=r.languages.extend("css",{comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},atrule:{pattern:/@[\w-](?:\([^()]+\)|[^()\s]|\s+(?!\s))*?(?=\s+[{;])/,inside:{rule:/@[\w-]+/}},url:/(?:[-a-z]+-)?url(?=\()/i,selector:{pattern:/(?=\S)[^@;{}()]?(?:[^@;{}()\s]|\s+(?!\s)|#\{\$[-\w]+\})+(?=\s*\{(?:\}|\s|[^}][^:{}]*[:{][^}]))/,inside:{parent:{pattern:/&/,alias:"important"},placeholder:/%[-\w]+/,variable:/\$[-\w]+|#\{\$[-\w]+\}/}},property:{pattern:/(?:[-\w]|\$[-\w]|#\{\$[-\w]+\})+(?=\s*:)/,inside:{variable:/\$[-\w]+|#\{\$[-\w]+\}/}}}),r.languages.insertBefore("scss","atrule",{keyword:[/@(?:content|debug|each|else(?: if)?|extend|for|forward|function|if|import|include|mixin|return|use|warn|while)\b/i,{pattern:/( )(?:from|through)(?= )/,lookbehind:!0}]}),r.languages.insertBefore("scss","important",{variable:/\$[-\w]+|#\{\$[-\w]+\}/}),r.languages.insertBefore("scss","function",{"module-modifier":{pattern:/\b(?:as|hide|show|with)\b/i,alias:"keyword"},placeholder:{pattern:/%[-\w]+/,alias:"selector"},statement:{pattern:/\B!(?:default|optional)\b/i,alias:"keyword"},boolean:/\b(?:false|true)\b/,null:{pattern:/\bnull\b/,alias:"keyword"},operator:{pattern:/(\s)(?:[-+*\/%]|[=!]=|<=?|>=?|and|not|or)(?=\s)/,lookbehind:!0}}),r.languages.scss.atrule.inside.rest=r.languages.scss,function(e){var t={pattern:/(\b\d+)(?:%|[a-z]+)/,lookbehind:!0},n={pattern:/(^|[^\w.-])-?(?:\d+(?:\.\d+)?|\.\d+)/,lookbehind:!0},s={comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0},url:{pattern:/\burl\((["']?).*?\1\)/i,greedy:!0},string:{pattern:/("|')(?:(?!\1)[^\\\r\n]|\\(?:\r\n|[\s\S]))*\1/,greedy:!0},interpolation:null,func:null,important:/\B!(?:important|optional)\b/i,keyword:{pattern:/(^|\s+)(?:(?:else|for|if|return|unless)(?=\s|$)|@[\w-]+)/,lookbehind:!0},hexcode:/#[\da-f]{3,6}/i,color:[/\b(?:AliceBlue|AntiqueWhite|Aqua|Aquamarine|Azure|Beige|Bisque|Black|BlanchedAlmond|Blue|BlueViolet|Brown|BurlyWood|CadetBlue|Chartreuse|Chocolate|Coral|CornflowerBlue|Cornsilk|Crimson|Cyan|DarkBlue|DarkCyan|DarkGoldenRod|DarkGr[ae]y|DarkGreen|DarkKhaki|DarkMagenta|DarkOliveGreen|DarkOrange|DarkOrchid|DarkRed|DarkSalmon|DarkSeaGreen|DarkSlateBlue|DarkSlateGr[ae]y|DarkTurquoise|DarkViolet|DeepPink|DeepSkyBlue|DimGr[ae]y|DodgerBlue|FireBrick|FloralWhite|ForestGreen|Fuchsia|Gainsboro|GhostWhite|Gold|GoldenRod|Gr[ae]y|Green|GreenYellow|HoneyDew|HotPink|IndianRed|Indigo|Ivory|Khaki|Lavender|LavenderBlush|LawnGreen|LemonChiffon|LightBlue|LightCoral|LightCyan|LightGoldenRodYellow|LightGr[ae]y|LightGreen|LightPink|LightSalmon|LightSeaGreen|LightSkyBlue|LightSlateGr[ae]y|LightSteelBlue|LightYellow|Lime|LimeGreen|Linen|Magenta|Maroon|MediumAquaMarine|MediumBlue|MediumOrchid|MediumPurple|MediumSeaGreen|MediumSlateBlue|MediumSpringGreen|MediumTurquoise|MediumVioletRed|MidnightBlue|MintCream|MistyRose|Moccasin|NavajoWhite|Navy|OldLace|Olive|OliveDrab|Orange|OrangeRed|Orchid|PaleGoldenRod|PaleGreen|PaleTurquoise|PaleVioletRed|PapayaWhip|PeachPuff|Peru|Pink|Plum|PowderBlue|Purple|Red|RosyBrown|RoyalBlue|SaddleBrown|Salmon|SandyBrown|SeaGreen|SeaShell|Sienna|Silver|SkyBlue|SlateBlue|SlateGr[ae]y|Snow|SpringGreen|SteelBlue|Tan|Teal|Thistle|Tomato|Transparent|Turquoise|Violet|Wheat|White|WhiteSmoke|Yellow|YellowGreen)\b/i,{pattern:/\b(?:hsl|rgb)\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*\)\B|\b(?:hsl|rgb)a\(\s*\d{1,3}\s*,\s*\d{1,3}%?\s*,\s*\d{1,3}%?\s*,\s*(?:0|0?\.\d+|1)\s*\)\B/i,inside:{unit:t,number:n,function:/[\w-]+(?=\()/,punctuation:/[(),]/}}],entity:/\\[\da-f]{1,8}/i,unit:t,boolean:/\b(?:false|true)\b/,operator:[/~|[+!\/%<>?=]=?|[-:]=|\*[*=]?|\.{2,3}|&&|\|\||\B-\B|\b(?:and|in|is(?: a| defined| not|nt)?|not|or)\b/],number:n,punctuation:/[{}()\[\];:,]/};s.interpolation={pattern:/\{[^\r\n}:]+\}/,alias:"variable",inside:{delimiter:{pattern:/^\{|\}$/,alias:"punctuation"},rest:s}},s.func={pattern:/[\w-]+\([^)]*\).*/,inside:{function:/^[^(]+/,rest:s}},e.languages.stylus={"atrule-declaration":{pattern:/(^[ \t]*)@.+/m,lookbehind:!0,inside:{atrule:/^@[\w-]+/,rest:s}},"variable-declaration":{pattern:/(^[ \t]*)[\w$-]+\s*.?=[ \t]*(?:\{[^{}]*\}|\S.*|$)/m,lookbehind:!0,inside:{variable:/^\S+/,rest:s}},statement:{pattern:/(^[ \t]*)(?:else|for|if|return|unless)[ \t].+/m,lookbehind:!0,inside:{keyword:/^\S+/,rest:s}},"property-declaration":{pattern:/((?:^|\{)([ \t]*))(?:[\w-]|\{[^}\r\n]+\})+(?:\s*:\s*|[ \t]+)(?!\s)[^{\r\n]*(?:;|[^{\r\n,]$(?!(?:\r?\n|\r)(?:\{|\2[ \t])))/m,lookbehind:!0,inside:{property:{pattern:/^[^\s:]+/,inside:{interpolation:s.interpolation}},rest:s}},selector:{pattern:/(^[ \t]*)(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)(?:(?:\r?\n|\r)(?:\1(?:(?=\S)(?:[^{}\r\n:()]|::?[\w-]+(?:\([^)\r\n]*\)|(?![\w-]))|\{[^}\r\n]+\})+)))*(?:,$|\{|(?=(?:\r?\n|\r)(?:\{|\1[ \t])))/m,lookbehind:!0,inside:{interpolation:s.interpolation,comment:s.comment,punctuation:/[{},]/}},func:s.func,string:s.string,comment:{pattern:/(^|[^\\])(?:\/\*[\s\S]*?\*\/|\/\/.*)/,lookbehind:!0,greedy:!0},interpolation:s.interpolation,punctuation:/[{}()\[\];:.]/}}(r),function(e){var t=e.util.clone(e.languages.typescript);e.languages.tsx=e.languages.extend("jsx",t),delete e.languages.tsx.parameter,delete e.languages.tsx["literal-property"];var n=e.languages.tsx.tag;n.pattern=RegExp(/(^|[^\w$]|(?=<\/))/.source+"(?:"+n.pattern.source+")",n.pattern.flags),n.lookbehind=!0}(r),r.languages.wasm={comment:[/\(;[\s\S]*?;\)/,{pattern:/;;.*/,greedy:!0}],string:{pattern:/"(?:\\[\s\S]|[^"\\])*"/,greedy:!0},keyword:[{pattern:/\b(?:align|offset)=/,inside:{operator:/=/}},{pattern:/\b(?:(?:f32|f64|i32|i64)(?:\.(?:abs|add|and|ceil|clz|const|convert_[su]\/i(?:32|64)|copysign|ctz|demote\/f64|div(?:_[su])?|eqz?|extend_[su]\/i32|floor|ge(?:_[su])?|gt(?:_[su])?|le(?:_[su])?|load(?:(?:8|16|32)_[su])?|lt(?:_[su])?|max|min|mul|neg?|nearest|or|popcnt|promote\/f32|reinterpret\/[fi](?:32|64)|rem_[su]|rot[lr]|shl|shr_[su]|sqrt|store(?:8|16|32)?|sub|trunc(?:_[su]\/f(?:32|64))?|wrap\/i64|xor))?|memory\.(?:grow|size))\b/,inside:{punctuation:/\./}},/\b(?:anyfunc|block|br(?:_if|_table)?|call(?:_indirect)?|data|drop|elem|else|end|export|func|get_(?:global|local)|global|if|import|local|loop|memory|module|mut|nop|offset|param|result|return|select|set_(?:global|local)|start|table|tee_local|then|type|unreachable)\b/],variable:/\$[\w!#$%&'*+\-./:<=>?@\\^`|~]+/,number:/[+-]?\b(?:\d(?:_?\d)*(?:\.\d(?:_?\d)*)?(?:[eE][+-]?\d(?:_?\d)*)?|0x[\da-fA-F](?:_?[\da-fA-F])*(?:\.[\da-fA-F](?:_?[\da-fA-D])*)?(?:[pP][+-]?\d(?:_?\d)*)?)\b|\binf\b|\bnan(?::0x[\da-fA-F](?:_?[\da-fA-D])*)?\b/,punctuation:/[()]/};const o=r},79016:()=>{!function(e){function t(e,t){return e.replace(/<<(\d+)>>/g,(function(e,n){return"(?:"+t[+n]+")"}))}function n(e,n,s){return RegExp(t(e,n),s||"")}function s(e,t){for(var n=0;n<t;n++)e=e.replace(/<<self>>/g,(function(){return"(?:"+e+")"}));return e.replace(/<<self>>/g,"[^\\s\\S]")}var r="bool byte char decimal double dynamic float int long object sbyte short string uint ulong ushort var void",o="class enum interface record struct",a="add alias and ascending async await by descending from(?=\\s*(?:\\w|$)) get global group into init(?=\\s*;) join let nameof not notnull on or orderby partial remove select set unmanaged value when where with(?=\\s*{)",i="abstract as base break case catch checked const continue default delegate do else event explicit extern finally fixed for foreach goto if implicit in internal is lock namespace new null operator out override params private protected public readonly ref return sealed sizeof stackalloc static switch this throw try typeof unchecked unsafe using virtual volatile while yield";function c(e){return"\\b(?:"+e.trim().replace(/ /g,"|")+")\\b"}var l=c(o),d=RegExp(c(r+" "+o+" "+a+" "+i)),p=c(o+" "+a+" "+i),u=c(r+" "+o+" "+i),m=s(/<(?:[^<>;=+\-*/%&|^]|<<self>>)*>/.source,2),h=s(/\((?:[^()]|<<self>>)*\)/.source,2),g=/@?\b[A-Za-z_]\w*\b/.source,f=t(/<<0>>(?:\s*<<1>>)?/.source,[g,m]),b=t(/(?!<<0>>)<<1>>(?:\s*\.\s*<<1>>)*/.source,[p,f]),y=/\[\s*(?:,\s*)*\]/.source,x=t(/<<0>>(?:\s*(?:\?\s*)?<<1>>)*(?:\s*\?)?/.source,[b,y]),v=t(/[^,()<>[\];=+\-*/%&|^]|<<0>>|<<1>>|<<2>>/.source,[m,h,y]),A=t(/\(<<0>>+(?:,<<0>>+)+\)/.source,[v]),S=t(/(?:<<0>>|<<1>>)(?:\s*(?:\?\s*)?<<2>>)*(?:\s*\?)?/.source,[A,b,y]),E={keyword:d,punctuation:/[<>()?,.:[\]]/},L=/'(?:[^\r\n'\\]|\\.|\\[Uux][\da-fA-F]{1,8})'/.source,M=/"(?:\\.|[^\\"\r\n])*"/.source,k=/@"(?:""|\\[\s\S]|[^\\"])*"(?!")/.source;e.languages.csharp=e.languages.extend("clike",{string:[{pattern:n(/(^|[^$\\])<<0>>/.source,[k]),lookbehind:!0,greedy:!0},{pattern:n(/(^|[^@$\\])<<0>>/.source,[M]),lookbehind:!0,greedy:!0}],"class-name":[{pattern:n(/(\busing\s+static\s+)<<0>>(?=\s*;)/.source,[b]),lookbehind:!0,inside:E},{pattern:n(/(\busing\s+<<0>>\s*=\s*)<<1>>(?=\s*;)/.source,[g,S]),lookbehind:!0,inside:E},{pattern:n(/(\busing\s+)<<0>>(?=\s*=)/.source,[g]),lookbehind:!0},{pattern:n(/(\b<<0>>\s+)<<1>>/.source,[l,f]),lookbehind:!0,inside:E},{pattern:n(/(\bcatch\s*\(\s*)<<0>>/.source,[b]),lookbehind:!0,inside:E},{pattern:n(/(\bwhere\s+)<<0>>/.source,[g]),lookbehind:!0},{pattern:n(/(\b(?:is(?:\s+not)?|as)\s+)<<0>>/.source,[x]),lookbehind:!0,inside:E},{pattern:n(/\b<<0>>(?=\s+(?!<<1>>|with\s*\{)<<2>>(?:\s*[=,;:{)\]]|\s+(?:in|when)\b))/.source,[S,u,g]),inside:E}],keyword:d,number:/(?:\b0(?:x[\da-f_]*[\da-f]|b[01_]*[01])|(?:\B\.\d+(?:_+\d+)*|\b\d+(?:_+\d+)*(?:\.\d+(?:_+\d+)*)?)(?:e[-+]?\d+(?:_+\d+)*)?)(?:[dflmu]|lu|ul)?\b/i,operator:/>>=?|<<=?|[-=]>|([-+&|])\1|~|\?\?=?|[-+*/%&|^!=<>]=?/,punctuation:/\?\.?|::|[{}[\];(),.:]/}),e.languages.insertBefore("csharp","number",{range:{pattern:/\.\./,alias:"operator"}}),e.languages.insertBefore("csharp","punctuation",{"named-parameter":{pattern:n(/([(,]\s*)<<0>>(?=\s*:)/.source,[g]),lookbehind:!0,alias:"punctuation"}}),e.languages.insertBefore("csharp","class-name",{namespace:{pattern:n(/(\b(?:namespace|using)\s+)<<0>>(?:\s*\.\s*<<0>>)*(?=\s*[;{])/.source,[g]),lookbehind:!0,inside:{punctuation:/\./}},"type-expression":{pattern:n(/(\b(?:default|sizeof|typeof)\s*\(\s*(?!\s))(?:[^()\s]|\s(?!\s)|<<0>>)*(?=\s*\))/.source,[h]),lookbehind:!0,alias:"class-name",inside:E},"return-type":{pattern:n(/<<0>>(?=\s+(?:<<1>>\s*(?:=>|[({]|\.\s*this\s*\[)|this\s*\[))/.source,[S,b]),inside:E,alias:"class-name"},"constructor-invocation":{pattern:n(/(\bnew\s+)<<0>>(?=\s*[[({])/.source,[S]),lookbehind:!0,inside:E,alias:"class-name"},"generic-method":{pattern:n(/<<0>>\s*<<1>>(?=\s*\()/.source,[g,m]),inside:{function:n(/^<<0>>/.source,[g]),generic:{pattern:RegExp(m),alias:"class-name",inside:E}}},"type-list":{pattern:n(/\b((?:<<0>>\s+<<1>>|record\s+<<1>>\s*<<5>>|where\s+<<2>>)\s*:\s*)(?:<<3>>|<<4>>|<<1>>\s*<<5>>|<<6>>)(?:\s*,\s*(?:<<3>>|<<4>>|<<6>>))*(?=\s*(?:where|[{;]|=>|$))/.source,[l,f,g,S,d.source,h,/\bnew\s*\(\s*\)/.source]),lookbehind:!0,inside:{"record-arguments":{pattern:n(/(^(?!new\s*\()<<0>>\s*)<<1>>/.source,[f,h]),lookbehind:!0,greedy:!0,inside:e.languages.csharp},keyword:d,"class-name":{pattern:RegExp(S),greedy:!0,inside:E},punctuation:/[,()]/}},preprocessor:{pattern:/(^[\t ]*)#.*/m,lookbehind:!0,alias:"property",inside:{directive:{pattern:/(#)\b(?:define|elif|else|endif|endregion|error|if|line|nullable|pragma|region|undef|warning)\b/,lookbehind:!0,alias:"keyword"}}}});var _=M+"|"+L,w=t(/\/(?![*/])|\/\/[^\r\n]*[\r\n]|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>/.source,[_]),I=s(t(/[^"'/()]|<<0>>|\(<<self>>*\)/.source,[w]),2),C=/\b(?:assembly|event|field|method|module|param|property|return|type)\b/.source,Q=t(/<<0>>(?:\s*\(<<1>>*\))?/.source,[b,I]);e.languages.insertBefore("csharp","class-name",{attribute:{pattern:n(/((?:^|[^\s\w>)?])\s*\[\s*)(?:<<0>>\s*:\s*)?<<1>>(?:\s*,\s*<<1>>)*(?=\s*\])/.source,[C,Q]),lookbehind:!0,greedy:!0,inside:{target:{pattern:n(/^<<0>>(?=\s*:)/.source,[C]),alias:"keyword"},"attribute-arguments":{pattern:n(/\(<<0>>*\)/.source,[I]),inside:e.languages.csharp},"class-name":{pattern:RegExp(b),inside:{punctuation:/\./}},punctuation:/[:,]/}}});var R=/:[^}\r\n]+/.source,D=s(t(/[^"'/()]|<<0>>|\(<<self>>*\)/.source,[w]),2),T=t(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source,[D,R]),O=s(t(/[^"'/()]|\/(?!\*)|\/\*(?:[^*]|\*(?!\/))*\*\/|<<0>>|\(<<self>>*\)/.source,[_]),2),P=t(/\{(?!\{)(?:(?![}:])<<0>>)*<<1>>?\}/.source,[O,R]);function F(t,s){return{interpolation:{pattern:n(/((?:^|[^{])(?:\{\{)*)<<0>>/.source,[t]),lookbehind:!0,inside:{"format-string":{pattern:n(/(^\{(?:(?![}:])<<0>>)*)<<1>>(?=\}$)/.source,[s,R]),lookbehind:!0,inside:{punctuation:/^:/}},punctuation:/^\{|\}$/,expression:{pattern:/[\s\S]+/,alias:"language-csharp",inside:e.languages.csharp}}},string:/[\s\S]+/}}e.languages.insertBefore("csharp","string",{"interpolation-string":[{pattern:n(/(^|[^\\])(?:\$@|@\$)"(?:""|\\[\s\S]|\{\{|<<0>>|[^\\{"])*"/.source,[T]),lookbehind:!0,greedy:!0,inside:F(T,D)},{pattern:n(/(^|[^@\\])\$"(?:\\.|\{\{|<<0>>|[^\\"{])*"/.source,[P]),lookbehind:!0,greedy:!0,inside:F(P,O)}],char:{pattern:RegExp(L),greedy:!0}}),e.languages.dotnet=e.languages.cs=e.languages.csharp}(Prism)},86862:()=>{!function(e){var t=e.languages.powershell={comment:[{pattern:/(^|[^`])<#[\s\S]*?#>/,lookbehind:!0},{pattern:/(^|[^`])#.*/,lookbehind:!0}],string:[{pattern:/"(?:`[\s\S]|[^`"])*"/,greedy:!0,inside:null},{pattern:/'(?:[^']|'')*'/,greedy:!0}],namespace:/\[[a-z](?:\[(?:\[[^\]]*\]|[^\[\]])*\]|[^\[\]])*\]/i,boolean:/\$(?:false|true)\b/i,variable:/\$\w+\b/,function:[/\b(?:Add|Approve|Assert|Backup|Block|Checkpoint|Clear|Close|Compare|Complete|Compress|Confirm|Connect|Convert|ConvertFrom|ConvertTo|Copy|Debug|Deny|Disable|Disconnect|Dismount|Edit|Enable|Enter|Exit|Expand|Export|Find|ForEach|Format|Get|Grant|Group|Hide|Import|Initialize|Install|Invoke|Join|Limit|Lock|Measure|Merge|Move|New|Open|Optimize|Out|Ping|Pop|Protect|Publish|Push|Read|Receive|Redo|Register|Remove|Rename|Repair|Request|Reset|Resize|Resolve|Restart|Restore|Resume|Revoke|Save|Search|Select|Send|Set|Show|Skip|Sort|Split|Start|Step|Stop|Submit|Suspend|Switch|Sync|Tee|Test|Trace|Unblock|Undo|Uninstall|Unlock|Unprotect|Unpublish|Unregister|Update|Use|Wait|Watch|Where|Write)-[a-z]+\b/i,/\b(?:ac|cat|chdir|clc|cli|clp|clv|compare|copy|cp|cpi|cpp|cvpa|dbp|del|diff|dir|ebp|echo|epal|epcsv|epsn|erase|fc|fl|ft|fw|gal|gbp|gc|gci|gcs|gdr|gi|gl|gm|gp|gps|group|gsv|gu|gv|gwmi|iex|ii|ipal|ipcsv|ipsn|irm|iwmi|iwr|kill|lp|ls|measure|mi|mount|move|mp|mv|nal|ndr|ni|nv|ogv|popd|ps|pushd|pwd|rbp|rd|rdr|ren|ri|rm|rmdir|rni|rnp|rp|rv|rvpa|rwmi|sal|saps|sasv|sbp|sc|select|set|shcm|si|sl|sleep|sls|sort|sp|spps|spsv|start|sv|swmi|tee|trcm|type|write)\b/i],keyword:/\b(?:Begin|Break|Catch|Class|Continue|Data|Define|Do|DynamicParam|Else|ElseIf|End|Exit|Filter|Finally|For|ForEach|From|Function|If|InlineScript|Parallel|Param|Process|Return|Sequence|Switch|Throw|Trap|Try|Until|Using|Var|While|Workflow)\b/i,operator:{pattern:/(^|\W)(?:!|-(?:b?(?:and|x?or)|as|(?:Not)?(?:Contains|In|Like|Match)|eq|ge|gt|is(?:Not)?|Join|le|lt|ne|not|Replace|sh[lr])\b|-[-=]?|\+[+=]?|[*\/%]=?)/i,lookbehind:!0},punctuation:/[|{}[\];(),.]/};t.string[0].inside={function:{pattern:/(^|[^`])\$\((?:\$\([^\r\n()]*\)|(?!\$\()[^\r\n)])*\)/,lookbehind:!0,inside:t},boolean:t.boolean,variable:t.variable}}(Prism)},55979:(e,t,n)=>{var s={"./prism-csharp":79016,"./prism-powershell":86862};function r(e){var t=o(e);return n(t)}function o(e){if(!n.o(s,e)){var t=new Error("Cannot find module '"+e+"'");throw t.code="MODULE_NOT_FOUND",t}return s[e]}r.keys=function(){return Object.keys(s)},r.resolve=o,e.exports=r,r.id=55979},92703:(e,t,n)=>{"use strict";var s=n(50414);function r(){}function o(){}o.resetWarningCache=r,e.exports=function(){function e(e,t,n,r,o,a){if(a!==s){var i=new Error("Calling PropTypes validators directly is not supported by the `prop-types` package. Use PropTypes.checkPropTypes() to call them. Read more at http://fb.me/use-check-prop-types");throw i.name="Invariant Violation",i}}function t(){return e}e.isRequired=e;var n={array:e,bigint:e,bool:e,func:e,number:e,object:e,string:e,symbol:e,any:e,arrayOf:t,element:e,elementType:e,instanceOf:t,node:e,objectOf:t,oneOf:t,oneOfType:t,shape:t,exact:t,checkPropTypes:o,resetWarningCache:r};return n.PropTypes=n,n}},45697:(e,t,n)=>{e.exports=n(92703)()},50414:e=>{"use strict";e.exports="SECRET_DO_NOT_PASS_THIS_OR_YOU_WILL_BE_FIRED"},64448:(e,t,n)=>{"use strict";var s=n(67294),r=n(27418),o=n(63840);function a(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n<arguments.length;n++)t+="&args[]="+encodeURIComponent(arguments[n]);return"Minified React error #"+e+"; visit "+t+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}if(!s)throw Error(a(227));function i(e,t,n,s,r,o,a,i,c){var l=Array.prototype.slice.call(arguments,3);try{t.apply(n,l)}catch(d){this.onError(d)}}var c=!1,l=null,d=!1,p=null,u={onError:function(e){c=!0,l=e}};function m(e,t,n,s,r,o,a,d,p){c=!1,l=null,i.apply(u,arguments)}var h=null,g=null,f=null;function b(e,t,n){var s=e.type||"unknown-event";e.currentTarget=f(n),function(e,t,n,s,r,o,i,u,h){if(m.apply(this,arguments),c){if(!c)throw Error(a(198));var g=l;c=!1,l=null,d||(d=!0,p=g)}}(s,t,void 0,e),e.currentTarget=null}var y=null,x={};function v(){if(y)for(var e in x){var t=x[e],n=y.indexOf(e);if(!(-1<n))throw Error(a(96,e));if(!S[n]){if(!t.extractEvents)throw Error(a(97,e));for(var s in S[n]=t,n=t.eventTypes){var r=void 0,o=n[s],i=t,c=s;if(E.hasOwnProperty(c))throw Error(a(99,c));E[c]=o;var l=o.phasedRegistrationNames;if(l){for(r in l)l.hasOwnProperty(r)&&A(l[r],i,c);r=!0}else o.registrationName?(A(o.registrationName,i,c),r=!0):r=!1;if(!r)throw Error(a(98,s,e))}}}}function A(e,t,n){if(L[e])throw Error(a(100,e));L[e]=t,M[e]=t.eventTypes[n].dependencies}var S=[],E={},L={},M={};function k(e){var t,n=!1;for(t in e)if(e.hasOwnProperty(t)){var s=e[t];if(!x.hasOwnProperty(t)||x[t]!==s){if(x[t])throw Error(a(102,t));x[t]=s,n=!0}}n&&v()}var _=!("undefined"==typeof window||void 0===window.document||void 0===window.document.createElement),w=null,I=null,C=null;function Q(e){if(e=g(e)){if("function"!=typeof w)throw Error(a(280));var t=e.stateNode;t&&(t=h(t),w(e.stateNode,e.type,t))}}function R(e){I?C?C.push(e):C=[e]:I=e}function D(){if(I){var e=I,t=C;if(C=I=null,Q(e),t)for(e=0;e<t.length;e++)Q(t[e])}}function T(e,t){return e(t)}function O(e,t,n,s,r){return e(t,n,s,r)}function P(){}var F=T,N=!1,B=!1;function V(){null===I&&null===C||(P(),D())}function G(e,t,n){if(B)return e(t,n);B=!0;try{return F(e,t,n)}finally{B=!1,V()}}var W=/^[:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD][:A-Z_a-z\u00C0-\u00D6\u00D8-\u00F6\u00F8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C-\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD\-.0-9\u00B7\u0300-\u036F\u203F-\u2040]*$/,U=Object.prototype.hasOwnProperty,z={},H={};function j(e,t,n,s,r,o){this.acceptsBooleans=2===t||3===t||4===t,this.attributeName=s,this.attributeNamespace=r,this.mustUseProperty=n,this.propertyName=e,this.type=t,this.sanitizeURL=o}var $={};"children dangerouslySetInnerHTML defaultValue defaultChecked innerHTML suppressContentEditableWarning suppressHydrationWarning style".split(" ").forEach((function(e){$[e]=new j(e,0,!1,e,null,!1)})),[["acceptCharset","accept-charset"],["className","class"],["htmlFor","for"],["httpEquiv","http-equiv"]].forEach((function(e){var t=e[0];$[t]=new j(t,1,!1,e[1],null,!1)})),["contentEditable","draggable","spellCheck","value"].forEach((function(e){$[e]=new j(e,2,!1,e.toLowerCase(),null,!1)})),["autoReverse","externalResourcesRequired","focusable","preserveAlpha"].forEach((function(e){$[e]=new j(e,2,!1,e,null,!1)})),"allowFullScreen async autoFocus autoPlay controls default defer disabled disablePictureInPicture formNoValidate hidden loop noModule noValidate open playsInline readOnly required reversed scoped seamless itemScope".split(" ").forEach((function(e){$[e]=new j(e,3,!1,e.toLowerCase(),null,!1)})),["checked","multiple","muted","selected"].forEach((function(e){$[e]=new j(e,3,!0,e,null,!1)})),["capture","download"].forEach((function(e){$[e]=new j(e,4,!1,e,null,!1)})),["cols","rows","size","span"].forEach((function(e){$[e]=new j(e,6,!1,e,null,!1)})),["rowSpan","start"].forEach((function(e){$[e]=new j(e,5,!1,e.toLowerCase(),null,!1)}));var Z=/[\-:]([a-z])/g;function X(e){return e[1].toUpperCase()}"accent-height alignment-baseline arabic-form baseline-shift cap-height clip-path clip-rule color-interpolation color-interpolation-filters color-profile color-rendering dominant-baseline enable-background fill-opacity fill-rule flood-color flood-opacity font-family font-size font-size-adjust font-stretch font-style font-variant font-weight glyph-name glyph-orientation-horizontal glyph-orientation-vertical horiz-adv-x horiz-origin-x image-rendering letter-spacing lighting-color marker-end marker-mid marker-start overline-position overline-thickness paint-order panose-1 pointer-events rendering-intent shape-rendering stop-color stop-opacity strikethrough-position strikethrough-thickness stroke-dasharray stroke-dashoffset stroke-linecap stroke-linejoin stroke-miterlimit stroke-opacity stroke-width text-anchor text-decoration text-rendering underline-position underline-thickness unicode-bidi unicode-range units-per-em v-alphabetic v-hanging v-ideographic v-mathematical vector-effect vert-adv-y vert-origin-x vert-origin-y word-spacing writing-mode xmlns:xlink x-height".split(" ").forEach((function(e){var t=e.replace(Z,X);$[t]=new j(t,1,!1,e,null,!1)})),"xlink:actuate xlink:arcrole xlink:role xlink:show xlink:title xlink:type".split(" ").forEach((function(e){var t=e.replace(Z,X);$[t]=new j(t,1,!1,e,"http://www.w3.org/1999/xlink",!1)})),["xml:base","xml:lang","xml:space"].forEach((function(e){var t=e.replace(Z,X);$[t]=new j(t,1,!1,e,"http://www.w3.org/XML/1998/namespace",!1)})),["tabIndex","crossOrigin"].forEach((function(e){$[e]=new j(e,1,!1,e.toLowerCase(),null,!1)})),$.xlinkHref=new j("xlinkHref",1,!1,"xlink:href","http://www.w3.org/1999/xlink",!0),["src","href","action","formAction"].forEach((function(e){$[e]=new j(e,1,!1,e.toLowerCase(),null,!0)}));var K=s.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED;function Y(e,t,n,s){var r=$.hasOwnProperty(t)?$[t]:null;(null!==r?0===r.type:!s&&(2<t.length&&("o"===t[0]||"O"===t[0])&&("n"===t[1]||"N"===t[1])))||(function(e,t,n,s){if(null==t||function(e,t,n,s){if(null!==n&&0===n.type)return!1;switch(typeof t){case"function":case"symbol":return!0;case"boolean":return!s&&(null!==n?!n.acceptsBooleans:"data-"!==(e=e.toLowerCase().slice(0,5))&&"aria-"!==e);default:return!1}}(e,t,n,s))return!0;if(s)return!1;if(null!==n)switch(n.type){case 3:return!t;case 4:return!1===t;case 5:return isNaN(t);case 6:return isNaN(t)||1>t}return!1}(t,n,r,s)&&(n=null),s||null===r?function(e){return!!U.call(H,e)||!U.call(z,e)&&(W.test(e)?H[e]=!0:(z[e]=!0,!1))}(t)&&(null===n?e.removeAttribute(t):e.setAttribute(t,""+n)):r.mustUseProperty?e[r.propertyName]=null===n?3!==r.type&&"":n:(t=r.attributeName,s=r.attributeNamespace,null===n?e.removeAttribute(t):(n=3===(r=r.type)||4===r&&!0===n?"":""+n,s?e.setAttributeNS(s,t,n):e.setAttribute(t,n))))}K.hasOwnProperty("ReactCurrentDispatcher")||(K.ReactCurrentDispatcher={current:null}),K.hasOwnProperty("ReactCurrentBatchConfig")||(K.ReactCurrentBatchConfig={suspense:null});var q=/^(.*)[\\\/]/,J="function"==typeof Symbol&&Symbol.for,ee=J?Symbol.for("react.element"):60103,te=J?Symbol.for("react.portal"):60106,ne=J?Symbol.for("react.fragment"):60107,se=J?Symbol.for("react.strict_mode"):60108,re=J?Symbol.for("react.profiler"):60114,oe=J?Symbol.for("react.provider"):60109,ae=J?Symbol.for("react.context"):60110,ie=J?Symbol.for("react.concurrent_mode"):60111,ce=J?Symbol.for("react.forward_ref"):60112,le=J?Symbol.for("react.suspense"):60113,de=J?Symbol.for("react.suspense_list"):60120,pe=J?Symbol.for("react.memo"):60115,ue=J?Symbol.for("react.lazy"):60116,me=J?Symbol.for("react.block"):60121,he="function"==typeof Symbol&&Symbol.iterator;function ge(e){return null===e||"object"!=typeof e?null:"function"==typeof(e=he&&e[he]||e["@@iterator"])?e:null}function fe(e){if(null==e)return null;if("function"==typeof e)return e.displayName||e.name||null;if("string"==typeof e)return e;switch(e){case ne:return"Fragment";case te:return"Portal";case re:return"Profiler";case se:return"StrictMode";case le:return"Suspense";case de:return"SuspenseList"}if("object"==typeof e)switch(e.$$typeof){case ae:return"Context.Consumer";case oe:return"Context.Provider";case ce:var t=e.render;return t=t.displayName||t.name||"",e.displayName||(""!==t?"ForwardRef("+t+")":"ForwardRef");case pe:return fe(e.type);case me:return fe(e.render);case ue:if(e=1===e._status?e._result:null)return fe(e)}return null}function be(e){var t="";do{e:switch(e.tag){case 3:case 4:case 6:case 7:case 10:case 9:var n="";break e;default:var s=e._debugOwner,r=e._debugSource,o=fe(e.type);n=null,s&&(n=fe(s.type)),s=o,o="",r?o=" (at "+r.fileName.replace(q,"")+":"+r.lineNumber+")":n&&(o=" (created by "+n+")"),n="\n    in "+(s||"Unknown")+o}t+=n,e=e.return}while(e);return t}function ye(e){switch(typeof e){case"boolean":case"number":case"object":case"string":case"undefined":return e;default:return""}}function xe(e){var t=e.type;return(e=e.nodeName)&&"input"===e.toLowerCase()&&("checkbox"===t||"radio"===t)}function ve(e){e._valueTracker||(e._valueTracker=function(e){var t=xe(e)?"checked":"value",n=Object.getOwnPropertyDescriptor(e.constructor.prototype,t),s=""+e[t];if(!e.hasOwnProperty(t)&&void 0!==n&&"function"==typeof n.get&&"function"==typeof n.set){var r=n.get,o=n.set;return Object.defineProperty(e,t,{configurable:!0,get:function(){return r.call(this)},set:function(e){s=""+e,o.call(this,e)}}),Object.defineProperty(e,t,{enumerable:n.enumerable}),{getValue:function(){return s},setValue:function(e){s=""+e},stopTracking:function(){e._valueTracker=null,delete e[t]}}}}(e))}function Ae(e){if(!e)return!1;var t=e._valueTracker;if(!t)return!0;var n=t.getValue(),s="";return e&&(s=xe(e)?e.checked?"true":"false":e.value),(e=s)!==n&&(t.setValue(e),!0)}function Se(e,t){var n=t.checked;return r({},t,{defaultChecked:void 0,defaultValue:void 0,value:void 0,checked:null!=n?n:e._wrapperState.initialChecked})}function Ee(e,t){var n=null==t.defaultValue?"":t.defaultValue,s=null!=t.checked?t.checked:t.defaultChecked;n=ye(null!=t.value?t.value:n),e._wrapperState={initialChecked:s,initialValue:n,controlled:"checkbox"===t.type||"radio"===t.type?null!=t.checked:null!=t.value}}function Le(e,t){null!=(t=t.checked)&&Y(e,"checked",t,!1)}function Me(e,t){Le(e,t);var n=ye(t.value),s=t.type;if(null!=n)"number"===s?(0===n&&""===e.value||e.value!=n)&&(e.value=""+n):e.value!==""+n&&(e.value=""+n);else if("submit"===s||"reset"===s)return void e.removeAttribute("value");t.hasOwnProperty("value")?_e(e,t.type,n):t.hasOwnProperty("defaultValue")&&_e(e,t.type,ye(t.defaultValue)),null==t.checked&&null!=t.defaultChecked&&(e.defaultChecked=!!t.defaultChecked)}function ke(e,t,n){if(t.hasOwnProperty("value")||t.hasOwnProperty("defaultValue")){var s=t.type;if(!("submit"!==s&&"reset"!==s||void 0!==t.value&&null!==t.value))return;t=""+e._wrapperState.initialValue,n||t===e.value||(e.value=t),e.defaultValue=t}""!==(n=e.name)&&(e.name=""),e.defaultChecked=!!e._wrapperState.initialChecked,""!==n&&(e.name=n)}function _e(e,t,n){"number"===t&&e.ownerDocument.activeElement===e||(null==n?e.defaultValue=""+e._wrapperState.initialValue:e.defaultValue!==""+n&&(e.defaultValue=""+n))}function we(e,t){return e=r({children:void 0},t),(t=function(e){var t="";return s.Children.forEach(e,(function(e){null!=e&&(t+=e)})),t}(t.children))&&(e.children=t),e}function Ie(e,t,n,s){if(e=e.options,t){t={};for(var r=0;r<n.length;r++)t["$"+n[r]]=!0;for(n=0;n<e.length;n++)r=t.hasOwnProperty("$"+e[n].value),e[n].selected!==r&&(e[n].selected=r),r&&s&&(e[n].defaultSelected=!0)}else{for(n=""+ye(n),t=null,r=0;r<e.length;r++){if(e[r].value===n)return e[r].selected=!0,void(s&&(e[r].defaultSelected=!0));null!==t||e[r].disabled||(t=e[r])}null!==t&&(t.selected=!0)}}function Ce(e,t){if(null!=t.dangerouslySetInnerHTML)throw Error(a(91));return r({},t,{value:void 0,defaultValue:void 0,children:""+e._wrapperState.initialValue})}function Qe(e,t){var n=t.value;if(null==n){if(n=t.children,t=t.defaultValue,null!=n){if(null!=t)throw Error(a(92));if(Array.isArray(n)){if(!(1>=n.length))throw Error(a(93));n=n[0]}t=n}null==t&&(t=""),n=t}e._wrapperState={initialValue:ye(n)}}function Re(e,t){var n=ye(t.value),s=ye(t.defaultValue);null!=n&&((n=""+n)!==e.value&&(e.value=n),null==t.defaultValue&&e.defaultValue!==n&&(e.defaultValue=n)),null!=s&&(e.defaultValue=""+s)}function De(e){var t=e.textContent;t===e._wrapperState.initialValue&&""!==t&&null!==t&&(e.value=t)}var Te="http://www.w3.org/1999/xhtml",Oe="http://www.w3.org/2000/svg";function Pe(e){switch(e){case"svg":return"http://www.w3.org/2000/svg";case"math":return"http://www.w3.org/1998/Math/MathML";default:return"http://www.w3.org/1999/xhtml"}}function Fe(e,t){return null==e||"http://www.w3.org/1999/xhtml"===e?Pe(t):"http://www.w3.org/2000/svg"===e&&"foreignObject"===t?"http://www.w3.org/1999/xhtml":e}var Ne,Be,Ve=(Be=function(e,t){if(e.namespaceURI!==Oe||"innerHTML"in e)e.innerHTML=t;else{for((Ne=Ne||document.createElement("div")).innerHTML="<svg>"+t.valueOf().toString()+"</svg>",t=Ne.firstChild;e.firstChild;)e.removeChild(e.firstChild);for(;t.firstChild;)e.appendChild(t.firstChild)}},"undefined"!=typeof MSApp&&MSApp.execUnsafeLocalFunction?function(e,t,n,s){MSApp.execUnsafeLocalFunction((function(){return Be(e,t)}))}:Be);function Ge(e,t){if(t){var n=e.firstChild;if(n&&n===e.lastChild&&3===n.nodeType)return void(n.nodeValue=t)}e.textContent=t}function We(e,t){var n={};return n[e.toLowerCase()]=t.toLowerCase(),n["Webkit"+e]="webkit"+t,n["Moz"+e]="moz"+t,n}var Ue={animationend:We("Animation","AnimationEnd"),animationiteration:We("Animation","AnimationIteration"),animationstart:We("Animation","AnimationStart"),transitionend:We("Transition","TransitionEnd")},ze={},He={};function je(e){if(ze[e])return ze[e];if(!Ue[e])return e;var t,n=Ue[e];for(t in n)if(n.hasOwnProperty(t)&&t in He)return ze[e]=n[t];return e}_&&(He=document.createElement("div").style,"AnimationEvent"in window||(delete Ue.animationend.animation,delete Ue.animationiteration.animation,delete Ue.animationstart.animation),"TransitionEvent"in window||delete Ue.transitionend.transition);var $e=je("animationend"),Ze=je("animationiteration"),Xe=je("animationstart"),Ke=je("transitionend"),Ye="abort canplay canplaythrough durationchange emptied encrypted ended error loadeddata loadedmetadata loadstart pause play playing progress ratechange seeked seeking stalled suspend timeupdate volumechange waiting".split(" "),qe=new("function"==typeof WeakMap?WeakMap:Map);function Je(e){var t=qe.get(e);return void 0===t&&(t=new Map,qe.set(e,t)),t}function et(e){var t=e,n=e;if(e.alternate)for(;t.return;)t=t.return;else{e=t;do{0!=(1026&(t=e).effectTag)&&(n=t.return),e=t.return}while(e)}return 3===t.tag?n:null}function tt(e){if(13===e.tag){var t=e.memoizedState;if(null===t&&(null!==(e=e.alternate)&&(t=e.memoizedState)),null!==t)return t.dehydrated}return null}function nt(e){if(et(e)!==e)throw Error(a(188))}function st(e){if(e=function(e){var t=e.alternate;if(!t){if(null===(t=et(e)))throw Error(a(188));return t!==e?null:e}for(var n=e,s=t;;){var r=n.return;if(null===r)break;var o=r.alternate;if(null===o){if(null!==(s=r.return)){n=s;continue}break}if(r.child===o.child){for(o=r.child;o;){if(o===n)return nt(r),e;if(o===s)return nt(r),t;o=o.sibling}throw Error(a(188))}if(n.return!==s.return)n=r,s=o;else{for(var i=!1,c=r.child;c;){if(c===n){i=!0,n=r,s=o;break}if(c===s){i=!0,s=r,n=o;break}c=c.sibling}if(!i){for(c=o.child;c;){if(c===n){i=!0,n=o,s=r;break}if(c===s){i=!0,s=o,n=r;break}c=c.sibling}if(!i)throw Error(a(189))}}if(n.alternate!==s)throw Error(a(190))}if(3!==n.tag)throw Error(a(188));return n.stateNode.current===n?e:t}(e),!e)return null;for(var t=e;;){if(5===t.tag||6===t.tag)return t;if(t.child)t.child.return=t,t=t.child;else{if(t===e)break;for(;!t.sibling;){if(!t.return||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}}return null}function rt(e,t){if(null==t)throw Error(a(30));return null==e?t:Array.isArray(e)?Array.isArray(t)?(e.push.apply(e,t),e):(e.push(t),e):Array.isArray(t)?[e].concat(t):[e,t]}function ot(e,t,n){Array.isArray(e)?e.forEach(t,n):e&&t.call(n,e)}var at=null;function it(e){if(e){var t=e._dispatchListeners,n=e._dispatchInstances;if(Array.isArray(t))for(var s=0;s<t.length&&!e.isPropagationStopped();s++)b(e,t[s],n[s]);else t&&b(e,t,n);e._dispatchListeners=null,e._dispatchInstances=null,e.isPersistent()||e.constructor.release(e)}}function ct(e){if(null!==e&&(at=rt(at,e)),e=at,at=null,e){if(ot(e,it),at)throw Error(a(95));if(d)throw e=p,d=!1,p=null,e}}function lt(e){return(e=e.target||e.srcElement||window).correspondingUseElement&&(e=e.correspondingUseElement),3===e.nodeType?e.parentNode:e}function dt(e){if(!_)return!1;var t=(e="on"+e)in document;return t||((t=document.createElement("div")).setAttribute(e,"return;"),t="function"==typeof t[e]),t}var pt=[];function ut(e){e.topLevelType=null,e.nativeEvent=null,e.targetInst=null,e.ancestors.length=0,10>pt.length&&pt.push(e)}function mt(e,t,n,s){if(pt.length){var r=pt.pop();return r.topLevelType=e,r.eventSystemFlags=s,r.nativeEvent=t,r.targetInst=n,r}return{topLevelType:e,eventSystemFlags:s,nativeEvent:t,targetInst:n,ancestors:[]}}function ht(e){var t=e.targetInst,n=t;do{if(!n){e.ancestors.push(n);break}var s=n;if(3===s.tag)s=s.stateNode.containerInfo;else{for(;s.return;)s=s.return;s=3!==s.tag?null:s.stateNode.containerInfo}if(!s)break;5!==(t=n.tag)&&6!==t||e.ancestors.push(n),n=Qn(s)}while(n);for(n=0;n<e.ancestors.length;n++){t=e.ancestors[n];var r=lt(e.nativeEvent);s=e.topLevelType;var o=e.nativeEvent,a=e.eventSystemFlags;0===n&&(a|=64);for(var i=null,c=0;c<S.length;c++){var l=S[c];l&&(l=l.extractEvents(s,t,o,r,a))&&(i=rt(i,l))}ct(i)}}function gt(e,t,n){if(!n.has(e)){switch(e){case"scroll":Xt(t,"scroll",!0);break;case"focus":case"blur":Xt(t,"focus",!0),Xt(t,"blur",!0),n.set("blur",null),n.set("focus",null);break;case"cancel":case"close":dt(e)&&Xt(t,e,!0);break;case"invalid":case"submit":case"reset":break;default:-1===Ye.indexOf(e)&&Zt(e,t)}n.set(e,null)}}var ft,bt,yt,xt=!1,vt=[],At=null,St=null,Et=null,Lt=new Map,Mt=new Map,kt=[],_t="mousedown mouseup touchcancel touchend touchstart auxclick dblclick pointercancel pointerdown pointerup dragend dragstart drop compositionend compositionstart keydown keypress keyup input textInput close cancel copy cut paste click change contextmenu reset submit".split(" "),wt="focus blur dragenter dragleave mouseover mouseout pointerover pointerout gotpointercapture lostpointercapture".split(" ");function It(e,t,n,s,r){return{blockedOn:e,topLevelType:t,eventSystemFlags:32|n,nativeEvent:r,container:s}}function Ct(e,t){switch(e){case"focus":case"blur":At=null;break;case"dragenter":case"dragleave":St=null;break;case"mouseover":case"mouseout":Et=null;break;case"pointerover":case"pointerout":Lt.delete(t.pointerId);break;case"gotpointercapture":case"lostpointercapture":Mt.delete(t.pointerId)}}function Qt(e,t,n,s,r,o){return null===e||e.nativeEvent!==o?(e=It(t,n,s,r,o),null!==t&&(null!==(t=Rn(t))&&bt(t)),e):(e.eventSystemFlags|=s,e)}function Rt(e){var t=Qn(e.target);if(null!==t){var n=et(t);if(null!==n)if(13===(t=n.tag)){if(null!==(t=tt(n)))return e.blockedOn=t,void o.unstable_runWithPriority(e.priority,(function(){yt(n)}))}else if(3===t&&n.stateNode.hydrate)return void(e.blockedOn=3===n.tag?n.stateNode.containerInfo:null)}e.blockedOn=null}function Dt(e){if(null!==e.blockedOn)return!1;var t=Jt(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);if(null!==t){var n=Rn(t);return null!==n&&bt(n),e.blockedOn=t,!1}return!0}function Tt(e,t,n){Dt(e)&&n.delete(t)}function Ot(){for(xt=!1;0<vt.length;){var e=vt[0];if(null!==e.blockedOn){null!==(e=Rn(e.blockedOn))&&ft(e);break}var t=Jt(e.topLevelType,e.eventSystemFlags,e.container,e.nativeEvent);null!==t?e.blockedOn=t:vt.shift()}null!==At&&Dt(At)&&(At=null),null!==St&&Dt(St)&&(St=null),null!==Et&&Dt(Et)&&(Et=null),Lt.forEach(Tt),Mt.forEach(Tt)}function Pt(e,t){e.blockedOn===t&&(e.blockedOn=null,xt||(xt=!0,o.unstable_scheduleCallback(o.unstable_NormalPriority,Ot)))}function Ft(e){function t(t){return Pt(t,e)}if(0<vt.length){Pt(vt[0],e);for(var n=1;n<vt.length;n++){var s=vt[n];s.blockedOn===e&&(s.blockedOn=null)}}for(null!==At&&Pt(At,e),null!==St&&Pt(St,e),null!==Et&&Pt(Et,e),Lt.forEach(t),Mt.forEach(t),n=0;n<kt.length;n++)(s=kt[n]).blockedOn===e&&(s.blockedOn=null);for(;0<kt.length&&null===(n=kt[0]).blockedOn;)Rt(n),null===n.blockedOn&&kt.shift()}var Nt={},Bt=new Map,Vt=new Map,Gt=["abort","abort",$e,"animationEnd",Ze,"animationIteration",Xe,"animationStart","canplay","canPlay","canplaythrough","canPlayThrough","durationchange","durationChange","emptied","emptied","encrypted","encrypted","ended","ended","error","error","gotpointercapture","gotPointerCapture","load","load","loadeddata","loadedData","loadedmetadata","loadedMetadata","loadstart","loadStart","lostpointercapture","lostPointerCapture","playing","playing","progress","progress","seeking","seeking","stalled","stalled","suspend","suspend","timeupdate","timeUpdate",Ke,"transitionEnd","waiting","waiting"];function Wt(e,t){for(var n=0;n<e.length;n+=2){var s=e[n],r=e[n+1],o="on"+(r[0].toUpperCase()+r.slice(1));o={phasedRegistrationNames:{bubbled:o,captured:o+"Capture"},dependencies:[s],eventPriority:t},Vt.set(s,t),Bt.set(s,o),Nt[r]=o}}Wt("blur blur cancel cancel click click close close contextmenu contextMenu copy copy cut cut auxclick auxClick dblclick doubleClick dragend dragEnd dragstart dragStart drop drop focus focus input input invalid invalid keydown keyDown keypress keyPress keyup keyUp mousedown mouseDown mouseup mouseUp paste paste pause pause play play pointercancel pointerCancel pointerdown pointerDown pointerup pointerUp ratechange rateChange reset reset seeked seeked submit submit touchcancel touchCancel touchend touchEnd touchstart touchStart volumechange volumeChange".split(" "),0),Wt("drag drag dragenter dragEnter dragexit dragExit dragleave dragLeave dragover dragOver mousemove mouseMove mouseout mouseOut mouseover mouseOver pointermove pointerMove pointerout pointerOut pointerover pointerOver scroll scroll toggle toggle touchmove touchMove wheel wheel".split(" "),1),Wt(Gt,2);for(var Ut="change selectionchange textInput compositionstart compositionend compositionupdate".split(" "),zt=0;zt<Ut.length;zt++)Vt.set(Ut[zt],0);var Ht=o.unstable_UserBlockingPriority,jt=o.unstable_runWithPriority,$t=!0;function Zt(e,t){Xt(t,e,!1)}function Xt(e,t,n){var s=Vt.get(t);switch(void 0===s?2:s){case 0:s=Kt.bind(null,t,1,e);break;case 1:s=Yt.bind(null,t,1,e);break;default:s=qt.bind(null,t,1,e)}n?e.addEventListener(t,s,!0):e.addEventListener(t,s,!1)}function Kt(e,t,n,s){N||P();var r=qt,o=N;N=!0;try{O(r,e,t,n,s)}finally{(N=o)||V()}}function Yt(e,t,n,s){jt(Ht,qt.bind(null,e,t,n,s))}function qt(e,t,n,s){if($t)if(0<vt.length&&-1<_t.indexOf(e))e=It(null,e,t,n,s),vt.push(e);else{var r=Jt(e,t,n,s);if(null===r)Ct(e,s);else if(-1<_t.indexOf(e))e=It(r,e,t,n,s),vt.push(e);else if(!function(e,t,n,s,r){switch(t){case"focus":return At=Qt(At,e,t,n,s,r),!0;case"dragenter":return St=Qt(St,e,t,n,s,r),!0;case"mouseover":return Et=Qt(Et,e,t,n,s,r),!0;case"pointerover":var o=r.pointerId;return Lt.set(o,Qt(Lt.get(o)||null,e,t,n,s,r)),!0;case"gotpointercapture":return o=r.pointerId,Mt.set(o,Qt(Mt.get(o)||null,e,t,n,s,r)),!0}return!1}(r,e,t,n,s)){Ct(e,s),e=mt(e,s,null,t);try{G(ht,e)}finally{ut(e)}}}}function Jt(e,t,n,s){if(null!==(n=Qn(n=lt(s)))){var r=et(n);if(null===r)n=null;else{var o=r.tag;if(13===o){if(null!==(n=tt(r)))return n;n=null}else if(3===o){if(r.stateNode.hydrate)return 3===r.tag?r.stateNode.containerInfo:null;n=null}else r!==n&&(n=null)}}e=mt(e,s,n,t);try{G(ht,e)}finally{ut(e)}return null}var en={animationIterationCount:!0,borderImageOutset:!0,borderImageSlice:!0,borderImageWidth:!0,boxFlex:!0,boxFlexGroup:!0,boxOrdinalGroup:!0,columnCount:!0,columns:!0,flex:!0,flexGrow:!0,flexPositive:!0,flexShrink:!0,flexNegative:!0,flexOrder:!0,gridArea:!0,gridRow:!0,gridRowEnd:!0,gridRowSpan:!0,gridRowStart:!0,gridColumn:!0,gridColumnEnd:!0,gridColumnSpan:!0,gridColumnStart:!0,fontWeight:!0,lineClamp:!0,lineHeight:!0,opacity:!0,order:!0,orphans:!0,tabSize:!0,widows:!0,zIndex:!0,zoom:!0,fillOpacity:!0,floodOpacity:!0,stopOpacity:!0,strokeDasharray:!0,strokeDashoffset:!0,strokeMiterlimit:!0,strokeOpacity:!0,strokeWidth:!0},tn=["Webkit","ms","Moz","O"];function nn(e,t,n){return null==t||"boolean"==typeof t||""===t?"":n||"number"!=typeof t||0===t||en.hasOwnProperty(e)&&en[e]?(""+t).trim():t+"px"}function sn(e,t){for(var n in e=e.style,t)if(t.hasOwnProperty(n)){var s=0===n.indexOf("--"),r=nn(n,t[n],s);"float"===n&&(n="cssFloat"),s?e.setProperty(n,r):e[n]=r}}Object.keys(en).forEach((function(e){tn.forEach((function(t){t=t+e.charAt(0).toUpperCase()+e.substring(1),en[t]=en[e]}))}));var rn=r({menuitem:!0},{area:!0,base:!0,br:!0,col:!0,embed:!0,hr:!0,img:!0,input:!0,keygen:!0,link:!0,meta:!0,param:!0,source:!0,track:!0,wbr:!0});function on(e,t){if(t){if(rn[e]&&(null!=t.children||null!=t.dangerouslySetInnerHTML))throw Error(a(137,e,""));if(null!=t.dangerouslySetInnerHTML){if(null!=t.children)throw Error(a(60));if("object"!=typeof t.dangerouslySetInnerHTML||!("__html"in t.dangerouslySetInnerHTML))throw Error(a(61))}if(null!=t.style&&"object"!=typeof t.style)throw Error(a(62,""))}}function an(e,t){if(-1===e.indexOf("-"))return"string"==typeof t.is;switch(e){case"annotation-xml":case"color-profile":case"font-face":case"font-face-src":case"font-face-uri":case"font-face-format":case"font-face-name":case"missing-glyph":return!1;default:return!0}}var cn=Te;function ln(e,t){var n=Je(e=9===e.nodeType||11===e.nodeType?e:e.ownerDocument);t=M[t];for(var s=0;s<t.length;s++)gt(t[s],e,n)}function dn(){}function pn(e){if(void 0===(e=e||("undefined"!=typeof document?document:void 0)))return null;try{return e.activeElement||e.body}catch(t){return e.body}}function un(e){for(;e&&e.firstChild;)e=e.firstChild;return e}function mn(e,t){var n,s=un(e);for(e=0;s;){if(3===s.nodeType){if(n=e+s.textContent.length,e<=t&&n>=t)return{node:s,offset:t-e};e=n}e:{for(;s;){if(s.nextSibling){s=s.nextSibling;break e}s=s.parentNode}s=void 0}s=un(s)}}function hn(e,t){return!(!e||!t)&&(e===t||(!e||3!==e.nodeType)&&(t&&3===t.nodeType?hn(e,t.parentNode):"contains"in e?e.contains(t):!!e.compareDocumentPosition&&!!(16&e.compareDocumentPosition(t))))}function gn(){for(var e=window,t=pn();t instanceof e.HTMLIFrameElement;){try{var n="string"==typeof t.contentWindow.location.href}catch(s){n=!1}if(!n)break;t=pn((e=t.contentWindow).document)}return t}function fn(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return t&&("input"===t&&("text"===e.type||"search"===e.type||"tel"===e.type||"url"===e.type||"password"===e.type)||"textarea"===t||"true"===e.contentEditable)}var bn="$?",yn="$!",xn=null,vn=null;function An(e,t){switch(e){case"button":case"input":case"select":case"textarea":return!!t.autoFocus}return!1}function Sn(e,t){return"textarea"===e||"option"===e||"noscript"===e||"string"==typeof t.children||"number"==typeof t.children||"object"==typeof t.dangerouslySetInnerHTML&&null!==t.dangerouslySetInnerHTML&&null!=t.dangerouslySetInnerHTML.__html}var En="function"==typeof setTimeout?setTimeout:void 0,Ln="function"==typeof clearTimeout?clearTimeout:void 0;function Mn(e){for(;null!=e;e=e.nextSibling){var t=e.nodeType;if(1===t||3===t)break}return e}function kn(e){e=e.previousSibling;for(var t=0;e;){if(8===e.nodeType){var n=e.data;if("$"===n||n===yn||n===bn){if(0===t)return e;t--}else"/$"===n&&t++}e=e.previousSibling}return null}var _n=Math.random().toString(36).slice(2),wn="__reactInternalInstance$"+_n,In="__reactEventHandlers$"+_n,Cn="__reactContainere$"+_n;function Qn(e){var t=e[wn];if(t)return t;for(var n=e.parentNode;n;){if(t=n[Cn]||n[wn]){if(n=t.alternate,null!==t.child||null!==n&&null!==n.child)for(e=kn(e);null!==e;){if(n=e[wn])return n;e=kn(e)}return t}n=(e=n).parentNode}return null}function Rn(e){return!(e=e[wn]||e[Cn])||5!==e.tag&&6!==e.tag&&13!==e.tag&&3!==e.tag?null:e}function Dn(e){if(5===e.tag||6===e.tag)return e.stateNode;throw Error(a(33))}function Tn(e){return e[In]||null}function On(e){do{e=e.return}while(e&&5!==e.tag);return e||null}function Pn(e,t){var n=e.stateNode;if(!n)return null;var s=h(n);if(!s)return null;n=s[t];e:switch(t){case"onClick":case"onClickCapture":case"onDoubleClick":case"onDoubleClickCapture":case"onMouseDown":case"onMouseDownCapture":case"onMouseMove":case"onMouseMoveCapture":case"onMouseUp":case"onMouseUpCapture":case"onMouseEnter":(s=!s.disabled)||(s=!("button"===(e=e.type)||"input"===e||"select"===e||"textarea"===e)),e=!s;break e;default:e=!1}if(e)return null;if(n&&"function"!=typeof n)throw Error(a(231,t,typeof n));return n}function Fn(e,t,n){(t=Pn(e,n.dispatchConfig.phasedRegistrationNames[t]))&&(n._dispatchListeners=rt(n._dispatchListeners,t),n._dispatchInstances=rt(n._dispatchInstances,e))}function Nn(e){if(e&&e.dispatchConfig.phasedRegistrationNames){for(var t=e._targetInst,n=[];t;)n.push(t),t=On(t);for(t=n.length;0<t--;)Fn(n[t],"captured",e);for(t=0;t<n.length;t++)Fn(n[t],"bubbled",e)}}function Bn(e,t,n){e&&n&&n.dispatchConfig.registrationName&&(t=Pn(e,n.dispatchConfig.registrationName))&&(n._dispatchListeners=rt(n._dispatchListeners,t),n._dispatchInstances=rt(n._dispatchInstances,e))}function Vn(e){e&&e.dispatchConfig.registrationName&&Bn(e._targetInst,null,e)}function Gn(e){ot(e,Nn)}var Wn=null,Un=null,zn=null;function Hn(){if(zn)return zn;var e,t,n=Un,s=n.length,r="value"in Wn?Wn.value:Wn.textContent,o=r.length;for(e=0;e<s&&n[e]===r[e];e++);var a=s-e;for(t=1;t<=a&&n[s-t]===r[o-t];t++);return zn=r.slice(e,1<t?1-t:void 0)}function jn(){return!0}function $n(){return!1}function Zn(e,t,n,s){for(var r in this.dispatchConfig=e,this._targetInst=t,this.nativeEvent=n,e=this.constructor.Interface)e.hasOwnProperty(r)&&((t=e[r])?this[r]=t(n):"target"===r?this.target=s:this[r]=n[r]);return this.isDefaultPrevented=(null!=n.defaultPrevented?n.defaultPrevented:!1===n.returnValue)?jn:$n,this.isPropagationStopped=$n,this}function Xn(e,t,n,s){if(this.eventPool.length){var r=this.eventPool.pop();return this.call(r,e,t,n,s),r}return new this(e,t,n,s)}function Kn(e){if(!(e instanceof this))throw Error(a(279));e.destructor(),10>this.eventPool.length&&this.eventPool.push(e)}function Yn(e){e.eventPool=[],e.getPooled=Xn,e.release=Kn}r(Zn.prototype,{preventDefault:function(){this.defaultPrevented=!0;var e=this.nativeEvent;e&&(e.preventDefault?e.preventDefault():"unknown"!=typeof e.returnValue&&(e.returnValue=!1),this.isDefaultPrevented=jn)},stopPropagation:function(){var e=this.nativeEvent;e&&(e.stopPropagation?e.stopPropagation():"unknown"!=typeof e.cancelBubble&&(e.cancelBubble=!0),this.isPropagationStopped=jn)},persist:function(){this.isPersistent=jn},isPersistent:$n,destructor:function(){var e,t=this.constructor.Interface;for(e in t)this[e]=null;this.nativeEvent=this._targetInst=this.dispatchConfig=null,this.isPropagationStopped=this.isDefaultPrevented=$n,this._dispatchInstances=this._dispatchListeners=null}}),Zn.Interface={type:null,target:null,currentTarget:function(){return null},eventPhase:null,bubbles:null,cancelable:null,timeStamp:function(e){return e.timeStamp||Date.now()},defaultPrevented:null,isTrusted:null},Zn.extend=function(e){function t(){}function n(){return s.apply(this,arguments)}var s=this;t.prototype=s.prototype;var o=new t;return r(o,n.prototype),n.prototype=o,n.prototype.constructor=n,n.Interface=r({},s.Interface,e),n.extend=s.extend,Yn(n),n},Yn(Zn);var qn=Zn.extend({data:null}),Jn=Zn.extend({data:null}),es=[9,13,27,32],ts=_&&"CompositionEvent"in window,ns=null;_&&"documentMode"in document&&(ns=document.documentMode);var ss=_&&"TextEvent"in window&&!ns,rs=_&&(!ts||ns&&8<ns&&11>=ns),os=String.fromCharCode(32),as={beforeInput:{phasedRegistrationNames:{bubbled:"onBeforeInput",captured:"onBeforeInputCapture"},dependencies:["compositionend","keypress","textInput","paste"]},compositionEnd:{phasedRegistrationNames:{bubbled:"onCompositionEnd",captured:"onCompositionEndCapture"},dependencies:"blur compositionend keydown keypress keyup mousedown".split(" ")},compositionStart:{phasedRegistrationNames:{bubbled:"onCompositionStart",captured:"onCompositionStartCapture"},dependencies:"blur compositionstart keydown keypress keyup mousedown".split(" ")},compositionUpdate:{phasedRegistrationNames:{bubbled:"onCompositionUpdate",captured:"onCompositionUpdateCapture"},dependencies:"blur compositionupdate keydown keypress keyup mousedown".split(" ")}},is=!1;function cs(e,t){switch(e){case"keyup":return-1!==es.indexOf(t.keyCode);case"keydown":return 229!==t.keyCode;case"keypress":case"mousedown":case"blur":return!0;default:return!1}}function ls(e){return"object"==typeof(e=e.detail)&&"data"in e?e.data:null}var ds=!1;var ps={eventTypes:as,extractEvents:function(e,t,n,s){var r;if(ts)e:{switch(e){case"compositionstart":var o=as.compositionStart;break e;case"compositionend":o=as.compositionEnd;break e;case"compositionupdate":o=as.compositionUpdate;break e}o=void 0}else ds?cs(e,n)&&(o=as.compositionEnd):"keydown"===e&&229===n.keyCode&&(o=as.compositionStart);return o?(rs&&"ko"!==n.locale&&(ds||o!==as.compositionStart?o===as.compositionEnd&&ds&&(r=Hn()):(Un="value"in(Wn=s)?Wn.value:Wn.textContent,ds=!0)),o=qn.getPooled(o,t,n,s),r?o.data=r:null!==(r=ls(n))&&(o.data=r),Gn(o),r=o):r=null,(e=ss?function(e,t){switch(e){case"compositionend":return ls(t);case"keypress":return 32!==t.which?null:(is=!0,os);case"textInput":return(e=t.data)===os&&is?null:e;default:return null}}(e,n):function(e,t){if(ds)return"compositionend"===e||!ts&&cs(e,t)?(e=Hn(),zn=Un=Wn=null,ds=!1,e):null;switch(e){case"paste":default:return null;case"keypress":if(!(t.ctrlKey||t.altKey||t.metaKey)||t.ctrlKey&&t.altKey){if(t.char&&1<t.char.length)return t.char;if(t.which)return String.fromCharCode(t.which)}return null;case"compositionend":return rs&&"ko"!==t.locale?null:t.data}}(e,n))?((t=Jn.getPooled(as.beforeInput,t,n,s)).data=e,Gn(t)):t=null,null===r?t:null===t?r:[r,t]}},us={color:!0,date:!0,datetime:!0,"datetime-local":!0,email:!0,month:!0,number:!0,password:!0,range:!0,search:!0,tel:!0,text:!0,time:!0,url:!0,week:!0};function ms(e){var t=e&&e.nodeName&&e.nodeName.toLowerCase();return"input"===t?!!us[e.type]:"textarea"===t}var hs={change:{phasedRegistrationNames:{bubbled:"onChange",captured:"onChangeCapture"},dependencies:"blur change click focus input keydown keyup selectionchange".split(" ")}};function gs(e,t,n){return(e=Zn.getPooled(hs.change,e,t,n)).type="change",R(n),Gn(e),e}var fs=null,bs=null;function ys(e){ct(e)}function xs(e){if(Ae(Dn(e)))return e}function vs(e,t){if("change"===e)return t}var As=!1;function Ss(){fs&&(fs.detachEvent("onpropertychange",Es),bs=fs=null)}function Es(e){if("value"===e.propertyName&&xs(bs))if(e=gs(bs,e,lt(e)),N)ct(e);else{N=!0;try{T(ys,e)}finally{N=!1,V()}}}function Ls(e,t,n){"focus"===e?(Ss(),bs=n,(fs=t).attachEvent("onpropertychange",Es)):"blur"===e&&Ss()}function Ms(e){if("selectionchange"===e||"keyup"===e||"keydown"===e)return xs(bs)}function ks(e,t){if("click"===e)return xs(t)}function _s(e,t){if("input"===e||"change"===e)return xs(t)}_&&(As=dt("input")&&(!document.documentMode||9<document.documentMode));var ws={eventTypes:hs,_isInputEventSupported:As,extractEvents:function(e,t,n,s){var r=t?Dn(t):window,o=r.nodeName&&r.nodeName.toLowerCase();if("select"===o||"input"===o&&"file"===r.type)var a=vs;else if(ms(r))if(As)a=_s;else{a=Ms;var i=Ls}else(o=r.nodeName)&&"input"===o.toLowerCase()&&("checkbox"===r.type||"radio"===r.type)&&(a=ks);if(a&&(a=a(e,t)))return gs(a,n,s);i&&i(e,r,t),"blur"===e&&(e=r._wrapperState)&&e.controlled&&"number"===r.type&&_e(r,"number",r.value)}},Is=Zn.extend({view:null,detail:null}),Cs={Alt:"altKey",Control:"ctrlKey",Meta:"metaKey",Shift:"shiftKey"};function Qs(e){var t=this.nativeEvent;return t.getModifierState?t.getModifierState(e):!!(e=Cs[e])&&!!t[e]}function Rs(){return Qs}var Ds=0,Ts=0,Os=!1,Ps=!1,Fs=Is.extend({screenX:null,screenY:null,clientX:null,clientY:null,pageX:null,pageY:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,getModifierState:Rs,button:null,buttons:null,relatedTarget:function(e){return e.relatedTarget||(e.fromElement===e.srcElement?e.toElement:e.fromElement)},movementX:function(e){if("movementX"in e)return e.movementX;var t=Ds;return Ds=e.screenX,Os?"mousemove"===e.type?e.screenX-t:0:(Os=!0,0)},movementY:function(e){if("movementY"in e)return e.movementY;var t=Ts;return Ts=e.screenY,Ps?"mousemove"===e.type?e.screenY-t:0:(Ps=!0,0)}}),Ns=Fs.extend({pointerId:null,width:null,height:null,pressure:null,tangentialPressure:null,tiltX:null,tiltY:null,twist:null,pointerType:null,isPrimary:null}),Bs={mouseEnter:{registrationName:"onMouseEnter",dependencies:["mouseout","mouseover"]},mouseLeave:{registrationName:"onMouseLeave",dependencies:["mouseout","mouseover"]},pointerEnter:{registrationName:"onPointerEnter",dependencies:["pointerout","pointerover"]},pointerLeave:{registrationName:"onPointerLeave",dependencies:["pointerout","pointerover"]}},Vs={eventTypes:Bs,extractEvents:function(e,t,n,s,r){var o="mouseover"===e||"pointerover"===e,a="mouseout"===e||"pointerout"===e;if(o&&0==(32&r)&&(n.relatedTarget||n.fromElement)||!a&&!o)return null;(o=s.window===s?s:(o=s.ownerDocument)?o.defaultView||o.parentWindow:window,a)?(a=t,null!==(t=(t=n.relatedTarget||n.toElement)?Qn(t):null)&&(t!==et(t)||5!==t.tag&&6!==t.tag)&&(t=null)):a=null;if(a===t)return null;if("mouseout"===e||"mouseover"===e)var i=Fs,c=Bs.mouseLeave,l=Bs.mouseEnter,d="mouse";else"pointerout"!==e&&"pointerover"!==e||(i=Ns,c=Bs.pointerLeave,l=Bs.pointerEnter,d="pointer");if(e=null==a?o:Dn(a),o=null==t?o:Dn(t),(c=i.getPooled(c,a,n,s)).type=d+"leave",c.target=e,c.relatedTarget=o,(n=i.getPooled(l,t,n,s)).type=d+"enter",n.target=o,n.relatedTarget=e,d=t,(s=a)&&d)e:{for(l=d,a=0,e=i=s;e;e=On(e))a++;for(e=0,t=l;t;t=On(t))e++;for(;0<a-e;)i=On(i),a--;for(;0<e-a;)l=On(l),e--;for(;a--;){if(i===l||i===l.alternate)break e;i=On(i),l=On(l)}i=null}else i=null;for(l=i,i=[];s&&s!==l&&(null===(a=s.alternate)||a!==l);)i.push(s),s=On(s);for(s=[];d&&d!==l&&(null===(a=d.alternate)||a!==l);)s.push(d),d=On(d);for(d=0;d<i.length;d++)Bn(i[d],"bubbled",c);for(d=s.length;0<d--;)Bn(s[d],"captured",n);return 0==(64&r)?[c]:[c,n]}};var Gs="function"==typeof Object.is?Object.is:function(e,t){return e===t&&(0!==e||1/e==1/t)||e!=e&&t!=t},Ws=Object.prototype.hasOwnProperty;function Us(e,t){if(Gs(e,t))return!0;if("object"!=typeof e||null===e||"object"!=typeof t||null===t)return!1;var n=Object.keys(e),s=Object.keys(t);if(n.length!==s.length)return!1;for(s=0;s<n.length;s++)if(!Ws.call(t,n[s])||!Gs(e[n[s]],t[n[s]]))return!1;return!0}var zs=_&&"documentMode"in document&&11>=document.documentMode,Hs={select:{phasedRegistrationNames:{bubbled:"onSelect",captured:"onSelectCapture"},dependencies:"blur contextmenu dragend focus keydown keyup mousedown mouseup selectionchange".split(" ")}},js=null,$s=null,Zs=null,Xs=!1;function Ks(e,t){var n=t.window===t?t.document:9===t.nodeType?t:t.ownerDocument;return Xs||null==js||js!==pn(n)?null:("selectionStart"in(n=js)&&fn(n)?n={start:n.selectionStart,end:n.selectionEnd}:n={anchorNode:(n=(n.ownerDocument&&n.ownerDocument.defaultView||window).getSelection()).anchorNode,anchorOffset:n.anchorOffset,focusNode:n.focusNode,focusOffset:n.focusOffset},Zs&&Us(Zs,n)?null:(Zs=n,(e=Zn.getPooled(Hs.select,$s,e,t)).type="select",e.target=js,Gn(e),e))}var Ys={eventTypes:Hs,extractEvents:function(e,t,n,s,r,o){if(!(o=!(r=o||(s.window===s?s.document:9===s.nodeType?s:s.ownerDocument)))){e:{r=Je(r),o=M.onSelect;for(var a=0;a<o.length;a++)if(!r.has(o[a])){r=!1;break e}r=!0}o=!r}if(o)return null;switch(r=t?Dn(t):window,e){case"focus":(ms(r)||"true"===r.contentEditable)&&(js=r,$s=t,Zs=null);break;case"blur":Zs=$s=js=null;break;case"mousedown":Xs=!0;break;case"contextmenu":case"mouseup":case"dragend":return Xs=!1,Ks(n,s);case"selectionchange":if(zs)break;case"keydown":case"keyup":return Ks(n,s)}return null}},qs=Zn.extend({animationName:null,elapsedTime:null,pseudoElement:null}),Js=Zn.extend({clipboardData:function(e){return"clipboardData"in e?e.clipboardData:window.clipboardData}}),er=Is.extend({relatedTarget:null});function tr(e){var t=e.keyCode;return"charCode"in e?0===(e=e.charCode)&&13===t&&(e=13):e=t,10===e&&(e=13),32<=e||13===e?e:0}var nr={Esc:"Escape",Spacebar:" ",Left:"ArrowLeft",Up:"ArrowUp",Right:"ArrowRight",Down:"ArrowDown",Del:"Delete",Win:"OS",Menu:"ContextMenu",Apps:"ContextMenu",Scroll:"ScrollLock",MozPrintableKey:"Unidentified"},sr={8:"Backspace",9:"Tab",12:"Clear",13:"Enter",16:"Shift",17:"Control",18:"Alt",19:"Pause",20:"CapsLock",27:"Escape",32:" ",33:"PageUp",34:"PageDown",35:"End",36:"Home",37:"ArrowLeft",38:"ArrowUp",39:"ArrowRight",40:"ArrowDown",45:"Insert",46:"Delete",112:"F1",113:"F2",114:"F3",115:"F4",116:"F5",117:"F6",118:"F7",119:"F8",120:"F9",121:"F10",122:"F11",123:"F12",144:"NumLock",145:"ScrollLock",224:"Meta"},rr=Is.extend({key:function(e){if(e.key){var t=nr[e.key]||e.key;if("Unidentified"!==t)return t}return"keypress"===e.type?13===(e=tr(e))?"Enter":String.fromCharCode(e):"keydown"===e.type||"keyup"===e.type?sr[e.keyCode]||"Unidentified":""},location:null,ctrlKey:null,shiftKey:null,altKey:null,metaKey:null,repeat:null,locale:null,getModifierState:Rs,charCode:function(e){return"keypress"===e.type?tr(e):0},keyCode:function(e){return"keydown"===e.type||"keyup"===e.type?e.keyCode:0},which:function(e){return"keypress"===e.type?tr(e):"keydown"===e.type||"keyup"===e.type?e.keyCode:0}}),or=Fs.extend({dataTransfer:null}),ar=Is.extend({touches:null,targetTouches:null,changedTouches:null,altKey:null,metaKey:null,ctrlKey:null,shiftKey:null,getModifierState:Rs}),ir=Zn.extend({propertyName:null,elapsedTime:null,pseudoElement:null}),cr=Fs.extend({deltaX:function(e){return"deltaX"in e?e.deltaX:"wheelDeltaX"in e?-e.wheelDeltaX:0},deltaY:function(e){return"deltaY"in e?e.deltaY:"wheelDeltaY"in e?-e.wheelDeltaY:"wheelDelta"in e?-e.wheelDelta:0},deltaZ:null,deltaMode:null}),lr={eventTypes:Nt,extractEvents:function(e,t,n,s){var r=Bt.get(e);if(!r)return null;switch(e){case"keypress":if(0===tr(n))return null;case"keydown":case"keyup":e=rr;break;case"blur":case"focus":e=er;break;case"click":if(2===n.button)return null;case"auxclick":case"dblclick":case"mousedown":case"mousemove":case"mouseup":case"mouseout":case"mouseover":case"contextmenu":e=Fs;break;case"drag":case"dragend":case"dragenter":case"dragexit":case"dragleave":case"dragover":case"dragstart":case"drop":e=or;break;case"touchcancel":case"touchend":case"touchmove":case"touchstart":e=ar;break;case $e:case Ze:case Xe:e=qs;break;case Ke:e=ir;break;case"scroll":e=Is;break;case"wheel":e=cr;break;case"copy":case"cut":case"paste":e=Js;break;case"gotpointercapture":case"lostpointercapture":case"pointercancel":case"pointerdown":case"pointermove":case"pointerout":case"pointerover":case"pointerup":e=Ns;break;default:e=Zn}return Gn(t=e.getPooled(r,t,n,s)),t}};if(y)throw Error(a(101));y=Array.prototype.slice.call("ResponderEventPlugin SimpleEventPlugin EnterLeaveEventPlugin ChangeEventPlugin SelectEventPlugin BeforeInputEventPlugin".split(" ")),v(),h=Tn,g=Rn,f=Dn,k({SimpleEventPlugin:lr,EnterLeaveEventPlugin:Vs,ChangeEventPlugin:ws,SelectEventPlugin:Ys,BeforeInputEventPlugin:ps});var dr=[],pr=-1;function ur(e){0>pr||(e.current=dr[pr],dr[pr]=null,pr--)}function mr(e,t){pr++,dr[pr]=e.current,e.current=t}var hr={},gr={current:hr},fr={current:!1},br=hr;function yr(e,t){var n=e.type.contextTypes;if(!n)return hr;var s=e.stateNode;if(s&&s.__reactInternalMemoizedUnmaskedChildContext===t)return s.__reactInternalMemoizedMaskedChildContext;var r,o={};for(r in n)o[r]=t[r];return s&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=t,e.__reactInternalMemoizedMaskedChildContext=o),o}function xr(e){return null!=(e=e.childContextTypes)}function vr(){ur(fr),ur(gr)}function Ar(e,t,n){if(gr.current!==hr)throw Error(a(168));mr(gr,t),mr(fr,n)}function Sr(e,t,n){var s=e.stateNode;if(e=t.childContextTypes,"function"!=typeof s.getChildContext)return n;for(var o in s=s.getChildContext())if(!(o in e))throw Error(a(108,fe(t)||"Unknown",o));return r({},n,{},s)}function Er(e){return e=(e=e.stateNode)&&e.__reactInternalMemoizedMergedChildContext||hr,br=gr.current,mr(gr,e),mr(fr,fr.current),!0}function Lr(e,t,n){var s=e.stateNode;if(!s)throw Error(a(169));n?(e=Sr(e,t,br),s.__reactInternalMemoizedMergedChildContext=e,ur(fr),ur(gr),mr(gr,e)):ur(fr),mr(fr,n)}var Mr=o.unstable_runWithPriority,kr=o.unstable_scheduleCallback,_r=o.unstable_cancelCallback,wr=o.unstable_requestPaint,Ir=o.unstable_now,Cr=o.unstable_getCurrentPriorityLevel,Qr=o.unstable_ImmediatePriority,Rr=o.unstable_UserBlockingPriority,Dr=o.unstable_NormalPriority,Tr=o.unstable_LowPriority,Or=o.unstable_IdlePriority,Pr={},Fr=o.unstable_shouldYield,Nr=void 0!==wr?wr:function(){},Br=null,Vr=null,Gr=!1,Wr=Ir(),Ur=1e4>Wr?Ir:function(){return Ir()-Wr};function zr(){switch(Cr()){case Qr:return 99;case Rr:return 98;case Dr:return 97;case Tr:return 96;case Or:return 95;default:throw Error(a(332))}}function Hr(e){switch(e){case 99:return Qr;case 98:return Rr;case 97:return Dr;case 96:return Tr;case 95:return Or;default:throw Error(a(332))}}function jr(e,t){return e=Hr(e),Mr(e,t)}function $r(e,t,n){return e=Hr(e),kr(e,t,n)}function Zr(e){return null===Br?(Br=[e],Vr=kr(Qr,Kr)):Br.push(e),Pr}function Xr(){if(null!==Vr){var e=Vr;Vr=null,_r(e)}Kr()}function Kr(){if(!Gr&&null!==Br){Gr=!0;var e=0;try{var t=Br;jr(99,(function(){for(;e<t.length;e++){var n=t[e];do{n=n(!0)}while(null!==n)}})),Br=null}catch(n){throw null!==Br&&(Br=Br.slice(e+1)),kr(Qr,Xr),n}finally{Gr=!1}}}function Yr(e,t,n){return 1073741821-(1+((1073741821-e+t/10)/(n/=10)|0))*n}function qr(e,t){if(e&&e.defaultProps)for(var n in t=r({},t),e=e.defaultProps)void 0===t[n]&&(t[n]=e[n]);return t}var Jr={current:null},eo=null,to=null,no=null;function so(){no=to=eo=null}function ro(e){var t=Jr.current;ur(Jr),e.type._context._currentValue=t}function oo(e,t){for(;null!==e;){var n=e.alternate;if(e.childExpirationTime<t)e.childExpirationTime=t,null!==n&&n.childExpirationTime<t&&(n.childExpirationTime=t);else{if(!(null!==n&&n.childExpirationTime<t))break;n.childExpirationTime=t}e=e.return}}function ao(e,t){eo=e,no=to=null,null!==(e=e.dependencies)&&null!==e.firstContext&&(e.expirationTime>=t&&(Ta=!0),e.firstContext=null)}function io(e,t){if(no!==e&&!1!==t&&0!==t)if("number"==typeof t&&1073741823!==t||(no=e,t=1073741823),t={context:e,observedBits:t,next:null},null===to){if(null===eo)throw Error(a(308));to=t,eo.dependencies={expirationTime:0,firstContext:t,responders:null}}else to=to.next=t;return e._currentValue}var co=!1;function lo(e){e.updateQueue={baseState:e.memoizedState,baseQueue:null,shared:{pending:null},effects:null}}function po(e,t){e=e.updateQueue,t.updateQueue===e&&(t.updateQueue={baseState:e.baseState,baseQueue:e.baseQueue,shared:e.shared,effects:e.effects})}function uo(e,t){return(e={expirationTime:e,suspenseConfig:t,tag:0,payload:null,callback:null,next:null}).next=e}function mo(e,t){if(null!==(e=e.updateQueue)){var n=(e=e.shared).pending;null===n?t.next=t:(t.next=n.next,n.next=t),e.pending=t}}function ho(e,t){var n=e.alternate;null!==n&&po(n,e),null===(n=(e=e.updateQueue).baseQueue)?(e.baseQueue=t.next=t,t.next=t):(t.next=n.next,n.next=t)}function go(e,t,n,s){var o=e.updateQueue;co=!1;var a=o.baseQueue,i=o.shared.pending;if(null!==i){if(null!==a){var c=a.next;a.next=i.next,i.next=c}a=i,o.shared.pending=null,null!==(c=e.alternate)&&(null!==(c=c.updateQueue)&&(c.baseQueue=i))}if(null!==a){c=a.next;var l=o.baseState,d=0,p=null,u=null,m=null;if(null!==c)for(var h=c;;){if((i=h.expirationTime)<s){var g={expirationTime:h.expirationTime,suspenseConfig:h.suspenseConfig,tag:h.tag,payload:h.payload,callback:h.callback,next:null};null===m?(u=m=g,p=l):m=m.next=g,i>d&&(d=i)}else{null!==m&&(m=m.next={expirationTime:1073741823,suspenseConfig:h.suspenseConfig,tag:h.tag,payload:h.payload,callback:h.callback,next:null}),mc(i,h.suspenseConfig);e:{var f=e,b=h;switch(i=t,g=n,b.tag){case 1:if("function"==typeof(f=b.payload)){l=f.call(g,l,i);break e}l=f;break e;case 3:f.effectTag=-4097&f.effectTag|64;case 0:if(null==(i="function"==typeof(f=b.payload)?f.call(g,l,i):f))break e;l=r({},l,i);break e;case 2:co=!0}}null!==h.callback&&(e.effectTag|=32,null===(i=o.effects)?o.effects=[h]:i.push(h))}if(null===(h=h.next)||h===c){if(null===(i=o.shared.pending))break;h=a.next=i.next,i.next=c,o.baseQueue=a=i,o.shared.pending=null}}null===m?p=l:m.next=u,o.baseState=p,o.baseQueue=m,hc(d),e.expirationTime=d,e.memoizedState=l}}function fo(e,t,n){if(e=t.effects,t.effects=null,null!==e)for(t=0;t<e.length;t++){var s=e[t],r=s.callback;if(null!==r){if(s.callback=null,s=r,r=n,"function"!=typeof s)throw Error(a(191,s));s.call(r)}}}var bo=K.ReactCurrentBatchConfig,yo=(new s.Component).refs;function xo(e,t,n,s){n=null==(n=n(s,t=e.memoizedState))?t:r({},t,n),e.memoizedState=n,0===e.expirationTime&&(e.updateQueue.baseState=n)}var vo={isMounted:function(e){return!!(e=e._reactInternalFiber)&&et(e)===e},enqueueSetState:function(e,t,n){e=e._reactInternalFiber;var s=ec(),r=bo.suspense;(r=uo(s=tc(s,e,r),r)).payload=t,null!=n&&(r.callback=n),mo(e,r),nc(e,s)},enqueueReplaceState:function(e,t,n){e=e._reactInternalFiber;var s=ec(),r=bo.suspense;(r=uo(s=tc(s,e,r),r)).tag=1,r.payload=t,null!=n&&(r.callback=n),mo(e,r),nc(e,s)},enqueueForceUpdate:function(e,t){e=e._reactInternalFiber;var n=ec(),s=bo.suspense;(s=uo(n=tc(n,e,s),s)).tag=2,null!=t&&(s.callback=t),mo(e,s),nc(e,n)}};function Ao(e,t,n,s,r,o,a){return"function"==typeof(e=e.stateNode).shouldComponentUpdate?e.shouldComponentUpdate(s,o,a):!t.prototype||!t.prototype.isPureReactComponent||(!Us(n,s)||!Us(r,o))}function So(e,t,n){var s=!1,r=hr,o=t.contextType;return"object"==typeof o&&null!==o?o=io(o):(r=xr(t)?br:gr.current,o=(s=null!=(s=t.contextTypes))?yr(e,r):hr),t=new t(n,o),e.memoizedState=null!==t.state&&void 0!==t.state?t.state:null,t.updater=vo,e.stateNode=t,t._reactInternalFiber=e,s&&((e=e.stateNode).__reactInternalMemoizedUnmaskedChildContext=r,e.__reactInternalMemoizedMaskedChildContext=o),t}function Eo(e,t,n,s){e=t.state,"function"==typeof t.componentWillReceiveProps&&t.componentWillReceiveProps(n,s),"function"==typeof t.UNSAFE_componentWillReceiveProps&&t.UNSAFE_componentWillReceiveProps(n,s),t.state!==e&&vo.enqueueReplaceState(t,t.state,null)}function Lo(e,t,n,s){var r=e.stateNode;r.props=n,r.state=e.memoizedState,r.refs=yo,lo(e);var o=t.contextType;"object"==typeof o&&null!==o?r.context=io(o):(o=xr(t)?br:gr.current,r.context=yr(e,o)),go(e,n,r,s),r.state=e.memoizedState,"function"==typeof(o=t.getDerivedStateFromProps)&&(xo(e,t,o,n),r.state=e.memoizedState),"function"==typeof t.getDerivedStateFromProps||"function"==typeof r.getSnapshotBeforeUpdate||"function"!=typeof r.UNSAFE_componentWillMount&&"function"!=typeof r.componentWillMount||(t=r.state,"function"==typeof r.componentWillMount&&r.componentWillMount(),"function"==typeof r.UNSAFE_componentWillMount&&r.UNSAFE_componentWillMount(),t!==r.state&&vo.enqueueReplaceState(r,r.state,null),go(e,n,r,s),r.state=e.memoizedState),"function"==typeof r.componentDidMount&&(e.effectTag|=4)}var Mo=Array.isArray;function ko(e,t,n){if(null!==(e=n.ref)&&"function"!=typeof e&&"object"!=typeof e){if(n._owner){if(n=n._owner){if(1!==n.tag)throw Error(a(309));var s=n.stateNode}if(!s)throw Error(a(147,e));var r=""+e;return null!==t&&null!==t.ref&&"function"==typeof t.ref&&t.ref._stringRef===r?t.ref:(t=function(e){var t=s.refs;t===yo&&(t=s.refs={}),null===e?delete t[r]:t[r]=e},t._stringRef=r,t)}if("string"!=typeof e)throw Error(a(284));if(!n._owner)throw Error(a(290,e))}return e}function _o(e,t){if("textarea"!==e.type)throw Error(a(31,"[object Object]"===Object.prototype.toString.call(t)?"object with keys {"+Object.keys(t).join(", ")+"}":t,""))}function wo(e){function t(t,n){if(e){var s=t.lastEffect;null!==s?(s.nextEffect=n,t.lastEffect=n):t.firstEffect=t.lastEffect=n,n.nextEffect=null,n.effectTag=8}}function n(n,s){if(!e)return null;for(;null!==s;)t(n,s),s=s.sibling;return null}function s(e,t){for(e=new Map;null!==t;)null!==t.key?e.set(t.key,t):e.set(t.index,t),t=t.sibling;return e}function r(e,t){return(e=Tc(e,t)).index=0,e.sibling=null,e}function o(t,n,s){return t.index=s,e?null!==(s=t.alternate)?(s=s.index)<n?(t.effectTag=2,n):s:(t.effectTag=2,n):n}function i(t){return e&&null===t.alternate&&(t.effectTag=2),t}function c(e,t,n,s){return null===t||6!==t.tag?((t=Fc(n,e.mode,s)).return=e,t):((t=r(t,n)).return=e,t)}function l(e,t,n,s){return null!==t&&t.elementType===n.type?((s=r(t,n.props)).ref=ko(e,t,n),s.return=e,s):((s=Oc(n.type,n.key,n.props,null,e.mode,s)).ref=ko(e,t,n),s.return=e,s)}function d(e,t,n,s){return null===t||4!==t.tag||t.stateNode.containerInfo!==n.containerInfo||t.stateNode.implementation!==n.implementation?((t=Nc(n,e.mode,s)).return=e,t):((t=r(t,n.children||[])).return=e,t)}function p(e,t,n,s,o){return null===t||7!==t.tag?((t=Pc(n,e.mode,s,o)).return=e,t):((t=r(t,n)).return=e,t)}function u(e,t,n){if("string"==typeof t||"number"==typeof t)return(t=Fc(""+t,e.mode,n)).return=e,t;if("object"==typeof t&&null!==t){switch(t.$$typeof){case ee:return(n=Oc(t.type,t.key,t.props,null,e.mode,n)).ref=ko(e,null,t),n.return=e,n;case te:return(t=Nc(t,e.mode,n)).return=e,t}if(Mo(t)||ge(t))return(t=Pc(t,e.mode,n,null)).return=e,t;_o(e,t)}return null}function m(e,t,n,s){var r=null!==t?t.key:null;if("string"==typeof n||"number"==typeof n)return null!==r?null:c(e,t,""+n,s);if("object"==typeof n&&null!==n){switch(n.$$typeof){case ee:return n.key===r?n.type===ne?p(e,t,n.props.children,s,r):l(e,t,n,s):null;case te:return n.key===r?d(e,t,n,s):null}if(Mo(n)||ge(n))return null!==r?null:p(e,t,n,s,null);_o(e,n)}return null}function h(e,t,n,s,r){if("string"==typeof s||"number"==typeof s)return c(t,e=e.get(n)||null,""+s,r);if("object"==typeof s&&null!==s){switch(s.$$typeof){case ee:return e=e.get(null===s.key?n:s.key)||null,s.type===ne?p(t,e,s.props.children,r,s.key):l(t,e,s,r);case te:return d(t,e=e.get(null===s.key?n:s.key)||null,s,r)}if(Mo(s)||ge(s))return p(t,e=e.get(n)||null,s,r,null);_o(t,s)}return null}function g(r,a,i,c){for(var l=null,d=null,p=a,g=a=0,f=null;null!==p&&g<i.length;g++){p.index>g?(f=p,p=null):f=p.sibling;var b=m(r,p,i[g],c);if(null===b){null===p&&(p=f);break}e&&p&&null===b.alternate&&t(r,p),a=o(b,a,g),null===d?l=b:d.sibling=b,d=b,p=f}if(g===i.length)return n(r,p),l;if(null===p){for(;g<i.length;g++)null!==(p=u(r,i[g],c))&&(a=o(p,a,g),null===d?l=p:d.sibling=p,d=p);return l}for(p=s(r,p);g<i.length;g++)null!==(f=h(p,r,g,i[g],c))&&(e&&null!==f.alternate&&p.delete(null===f.key?g:f.key),a=o(f,a,g),null===d?l=f:d.sibling=f,d=f);return e&&p.forEach((function(e){return t(r,e)})),l}function f(r,i,c,l){var d=ge(c);if("function"!=typeof d)throw Error(a(150));if(null==(c=d.call(c)))throw Error(a(151));for(var p=d=null,g=i,f=i=0,b=null,y=c.next();null!==g&&!y.done;f++,y=c.next()){g.index>f?(b=g,g=null):b=g.sibling;var x=m(r,g,y.value,l);if(null===x){null===g&&(g=b);break}e&&g&&null===x.alternate&&t(r,g),i=o(x,i,f),null===p?d=x:p.sibling=x,p=x,g=b}if(y.done)return n(r,g),d;if(null===g){for(;!y.done;f++,y=c.next())null!==(y=u(r,y.value,l))&&(i=o(y,i,f),null===p?d=y:p.sibling=y,p=y);return d}for(g=s(r,g);!y.done;f++,y=c.next())null!==(y=h(g,r,f,y.value,l))&&(e&&null!==y.alternate&&g.delete(null===y.key?f:y.key),i=o(y,i,f),null===p?d=y:p.sibling=y,p=y);return e&&g.forEach((function(e){return t(r,e)})),d}return function(e,s,o,c){var l="object"==typeof o&&null!==o&&o.type===ne&&null===o.key;l&&(o=o.props.children);var d="object"==typeof o&&null!==o;if(d)switch(o.$$typeof){case ee:e:{for(d=o.key,l=s;null!==l;){if(l.key===d){if(7===l.tag){if(o.type===ne){n(e,l.sibling),(s=r(l,o.props.children)).return=e,e=s;break e}}else if(l.elementType===o.type){n(e,l.sibling),(s=r(l,o.props)).ref=ko(e,l,o),s.return=e,e=s;break e}n(e,l);break}t(e,l),l=l.sibling}o.type===ne?((s=Pc(o.props.children,e.mode,c,o.key)).return=e,e=s):((c=Oc(o.type,o.key,o.props,null,e.mode,c)).ref=ko(e,s,o),c.return=e,e=c)}return i(e);case te:e:{for(l=o.key;null!==s;){if(s.key===l){if(4===s.tag&&s.stateNode.containerInfo===o.containerInfo&&s.stateNode.implementation===o.implementation){n(e,s.sibling),(s=r(s,o.children||[])).return=e,e=s;break e}n(e,s);break}t(e,s),s=s.sibling}(s=Nc(o,e.mode,c)).return=e,e=s}return i(e)}if("string"==typeof o||"number"==typeof o)return o=""+o,null!==s&&6===s.tag?(n(e,s.sibling),(s=r(s,o)).return=e,e=s):(n(e,s),(s=Fc(o,e.mode,c)).return=e,e=s),i(e);if(Mo(o))return g(e,s,o,c);if(ge(o))return f(e,s,o,c);if(d&&_o(e,o),void 0===o&&!l)switch(e.tag){case 1:case 0:throw e=e.type,Error(a(152,e.displayName||e.name||"Component"))}return n(e,s)}}var Io=wo(!0),Co=wo(!1),Qo={},Ro={current:Qo},Do={current:Qo},To={current:Qo};function Oo(e){if(e===Qo)throw Error(a(174));return e}function Po(e,t){switch(mr(To,t),mr(Do,e),mr(Ro,Qo),e=t.nodeType){case 9:case 11:t=(t=t.documentElement)?t.namespaceURI:Fe(null,"");break;default:t=Fe(t=(e=8===e?t.parentNode:t).namespaceURI||null,e=e.tagName)}ur(Ro),mr(Ro,t)}function Fo(){ur(Ro),ur(Do),ur(To)}function No(e){Oo(To.current);var t=Oo(Ro.current),n=Fe(t,e.type);t!==n&&(mr(Do,e),mr(Ro,n))}function Bo(e){Do.current===e&&(ur(Ro),ur(Do))}var Vo={current:0};function Go(e){for(var t=e;null!==t;){if(13===t.tag){var n=t.memoizedState;if(null!==n&&(null===(n=n.dehydrated)||n.data===bn||n.data===yn))return t}else if(19===t.tag&&void 0!==t.memoizedProps.revealOrder){if(0!=(64&t.effectTag))return t}else if(null!==t.child){t.child.return=t,t=t.child;continue}if(t===e)break;for(;null===t.sibling;){if(null===t.return||t.return===e)return null;t=t.return}t.sibling.return=t.return,t=t.sibling}return null}function Wo(e,t){return{responder:e,props:t}}var Uo=K.ReactCurrentDispatcher,zo=K.ReactCurrentBatchConfig,Ho=0,jo=null,$o=null,Zo=null,Xo=!1;function Ko(){throw Error(a(321))}function Yo(e,t){if(null===t)return!1;for(var n=0;n<t.length&&n<e.length;n++)if(!Gs(e[n],t[n]))return!1;return!0}function qo(e,t,n,s,r,o){if(Ho=o,jo=t,t.memoizedState=null,t.updateQueue=null,t.expirationTime=0,Uo.current=null===e||null===e.memoizedState?Aa:Sa,e=n(s,r),t.expirationTime===Ho){o=0;do{if(t.expirationTime=0,!(25>o))throw Error(a(301));o+=1,Zo=$o=null,t.updateQueue=null,Uo.current=Ea,e=n(s,r)}while(t.expirationTime===Ho)}if(Uo.current=va,t=null!==$o&&null!==$o.next,Ho=0,Zo=$o=jo=null,Xo=!1,t)throw Error(a(300));return e}function Jo(){var e={memoizedState:null,baseState:null,baseQueue:null,queue:null,next:null};return null===Zo?jo.memoizedState=Zo=e:Zo=Zo.next=e,Zo}function ea(){if(null===$o){var e=jo.alternate;e=null!==e?e.memoizedState:null}else e=$o.next;var t=null===Zo?jo.memoizedState:Zo.next;if(null!==t)Zo=t,$o=e;else{if(null===e)throw Error(a(310));e={memoizedState:($o=e).memoizedState,baseState:$o.baseState,baseQueue:$o.baseQueue,queue:$o.queue,next:null},null===Zo?jo.memoizedState=Zo=e:Zo=Zo.next=e}return Zo}function ta(e,t){return"function"==typeof t?t(e):t}function na(e){var t=ea(),n=t.queue;if(null===n)throw Error(a(311));n.lastRenderedReducer=e;var s=$o,r=s.baseQueue,o=n.pending;if(null!==o){if(null!==r){var i=r.next;r.next=o.next,o.next=i}s.baseQueue=r=o,n.pending=null}if(null!==r){r=r.next,s=s.baseState;var c=i=o=null,l=r;do{var d=l.expirationTime;if(d<Ho){var p={expirationTime:l.expirationTime,suspenseConfig:l.suspenseConfig,action:l.action,eagerReducer:l.eagerReducer,eagerState:l.eagerState,next:null};null===c?(i=c=p,o=s):c=c.next=p,d>jo.expirationTime&&(jo.expirationTime=d,hc(d))}else null!==c&&(c=c.next={expirationTime:1073741823,suspenseConfig:l.suspenseConfig,action:l.action,eagerReducer:l.eagerReducer,eagerState:l.eagerState,next:null}),mc(d,l.suspenseConfig),s=l.eagerReducer===e?l.eagerState:e(s,l.action);l=l.next}while(null!==l&&l!==r);null===c?o=s:c.next=i,Gs(s,t.memoizedState)||(Ta=!0),t.memoizedState=s,t.baseState=o,t.baseQueue=c,n.lastRenderedState=s}return[t.memoizedState,n.dispatch]}function sa(e){var t=ea(),n=t.queue;if(null===n)throw Error(a(311));n.lastRenderedReducer=e;var s=n.dispatch,r=n.pending,o=t.memoizedState;if(null!==r){n.pending=null;var i=r=r.next;do{o=e(o,i.action),i=i.next}while(i!==r);Gs(o,t.memoizedState)||(Ta=!0),t.memoizedState=o,null===t.baseQueue&&(t.baseState=o),n.lastRenderedState=o}return[o,s]}function ra(e){var t=Jo();return"function"==typeof e&&(e=e()),t.memoizedState=t.baseState=e,e=(e=t.queue={pending:null,dispatch:null,lastRenderedReducer:ta,lastRenderedState:e}).dispatch=xa.bind(null,jo,e),[t.memoizedState,e]}function oa(e,t,n,s){return e={tag:e,create:t,destroy:n,deps:s,next:null},null===(t=jo.updateQueue)?(t={lastEffect:null},jo.updateQueue=t,t.lastEffect=e.next=e):null===(n=t.lastEffect)?t.lastEffect=e.next=e:(s=n.next,n.next=e,e.next=s,t.lastEffect=e),e}function aa(){return ea().memoizedState}function ia(e,t,n,s){var r=Jo();jo.effectTag|=e,r.memoizedState=oa(1|t,n,void 0,void 0===s?null:s)}function ca(e,t,n,s){var r=ea();s=void 0===s?null:s;var o=void 0;if(null!==$o){var a=$o.memoizedState;if(o=a.destroy,null!==s&&Yo(s,a.deps))return void oa(t,n,o,s)}jo.effectTag|=e,r.memoizedState=oa(1|t,n,o,s)}function la(e,t){return ia(516,4,e,t)}function da(e,t){return ca(516,4,e,t)}function pa(e,t){return ca(4,2,e,t)}function ua(e,t){return"function"==typeof t?(e=e(),t(e),function(){t(null)}):null!=t?(e=e(),t.current=e,function(){t.current=null}):void 0}function ma(e,t,n){return n=null!=n?n.concat([e]):null,ca(4,2,ua.bind(null,t,e),n)}function ha(){}function ga(e,t){return Jo().memoizedState=[e,void 0===t?null:t],e}function fa(e,t){var n=ea();t=void 0===t?null:t;var s=n.memoizedState;return null!==s&&null!==t&&Yo(t,s[1])?s[0]:(n.memoizedState=[e,t],e)}function ba(e,t){var n=ea();t=void 0===t?null:t;var s=n.memoizedState;return null!==s&&null!==t&&Yo(t,s[1])?s[0]:(e=e(),n.memoizedState=[e,t],e)}function ya(e,t,n){var s=zr();jr(98>s?98:s,(function(){e(!0)})),jr(97<s?97:s,(function(){var s=zo.suspense;zo.suspense=void 0===t?null:t;try{e(!1),n()}finally{zo.suspense=s}}))}function xa(e,t,n){var s=ec(),r=bo.suspense;r={expirationTime:s=tc(s,e,r),suspenseConfig:r,action:n,eagerReducer:null,eagerState:null,next:null};var o=t.pending;if(null===o?r.next=r:(r.next=o.next,o.next=r),t.pending=r,o=e.alternate,e===jo||null!==o&&o===jo)Xo=!0,r.expirationTime=Ho,jo.expirationTime=Ho;else{if(0===e.expirationTime&&(null===o||0===o.expirationTime)&&null!==(o=t.lastRenderedReducer))try{var a=t.lastRenderedState,i=o(a,n);if(r.eagerReducer=o,r.eagerState=i,Gs(i,a))return}catch(c){}nc(e,s)}}var va={readContext:io,useCallback:Ko,useContext:Ko,useEffect:Ko,useImperativeHandle:Ko,useLayoutEffect:Ko,useMemo:Ko,useReducer:Ko,useRef:Ko,useState:Ko,useDebugValue:Ko,useResponder:Ko,useDeferredValue:Ko,useTransition:Ko},Aa={readContext:io,useCallback:ga,useContext:io,useEffect:la,useImperativeHandle:function(e,t,n){return n=null!=n?n.concat([e]):null,ia(4,2,ua.bind(null,t,e),n)},useLayoutEffect:function(e,t){return ia(4,2,e,t)},useMemo:function(e,t){var n=Jo();return t=void 0===t?null:t,e=e(),n.memoizedState=[e,t],e},useReducer:function(e,t,n){var s=Jo();return t=void 0!==n?n(t):t,s.memoizedState=s.baseState=t,e=(e=s.queue={pending:null,dispatch:null,lastRenderedReducer:e,lastRenderedState:t}).dispatch=xa.bind(null,jo,e),[s.memoizedState,e]},useRef:function(e){return e={current:e},Jo().memoizedState=e},useState:ra,useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=ra(e),s=n[0],r=n[1];return la((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=ra(!1),n=t[0];return t=t[1],[ga(ya.bind(null,t,e),[t,e]),n]}},Sa={readContext:io,useCallback:fa,useContext:io,useEffect:da,useImperativeHandle:ma,useLayoutEffect:pa,useMemo:ba,useReducer:na,useRef:aa,useState:function(){return na(ta)},useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=na(ta),s=n[0],r=n[1];return da((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=na(ta),n=t[0];return t=t[1],[fa(ya.bind(null,t,e),[t,e]),n]}},Ea={readContext:io,useCallback:fa,useContext:io,useEffect:da,useImperativeHandle:ma,useLayoutEffect:pa,useMemo:ba,useReducer:sa,useRef:aa,useState:function(){return sa(ta)},useDebugValue:ha,useResponder:Wo,useDeferredValue:function(e,t){var n=sa(ta),s=n[0],r=n[1];return da((function(){var n=zo.suspense;zo.suspense=void 0===t?null:t;try{r(e)}finally{zo.suspense=n}}),[e,t]),s},useTransition:function(e){var t=sa(ta),n=t[0];return t=t[1],[fa(ya.bind(null,t,e),[t,e]),n]}},La=null,Ma=null,ka=!1;function _a(e,t){var n=Rc(5,null,null,0);n.elementType="DELETED",n.type="DELETED",n.stateNode=t,n.return=e,n.effectTag=8,null!==e.lastEffect?(e.lastEffect.nextEffect=n,e.lastEffect=n):e.firstEffect=e.lastEffect=n}function wa(e,t){switch(e.tag){case 5:var n=e.type;return null!==(t=1!==t.nodeType||n.toLowerCase()!==t.nodeName.toLowerCase()?null:t)&&(e.stateNode=t,!0);case 6:return null!==(t=""===e.pendingProps||3!==t.nodeType?null:t)&&(e.stateNode=t,!0);default:return!1}}function Ia(e){if(ka){var t=Ma;if(t){var n=t;if(!wa(e,t)){if(!(t=Mn(n.nextSibling))||!wa(e,t))return e.effectTag=-1025&e.effectTag|2,ka=!1,void(La=e);_a(La,n)}La=e,Ma=Mn(t.firstChild)}else e.effectTag=-1025&e.effectTag|2,ka=!1,La=e}}function Ca(e){for(e=e.return;null!==e&&5!==e.tag&&3!==e.tag&&13!==e.tag;)e=e.return;La=e}function Qa(e){if(e!==La)return!1;if(!ka)return Ca(e),ka=!0,!1;var t=e.type;if(5!==e.tag||"head"!==t&&"body"!==t&&!Sn(t,e.memoizedProps))for(t=Ma;t;)_a(e,t),t=Mn(t.nextSibling);if(Ca(e),13===e.tag){if(!(e=null!==(e=e.memoizedState)?e.dehydrated:null))throw Error(a(317));e:{for(e=e.nextSibling,t=0;e;){if(8===e.nodeType){var n=e.data;if("/$"===n){if(0===t){Ma=Mn(e.nextSibling);break e}t--}else"$"!==n&&n!==yn&&n!==bn||t++}e=e.nextSibling}Ma=null}}else Ma=La?Mn(e.stateNode.nextSibling):null;return!0}function Ra(){Ma=La=null,ka=!1}var Da=K.ReactCurrentOwner,Ta=!1;function Oa(e,t,n,s){t.child=null===e?Co(t,null,n,s):Io(t,e.child,n,s)}function Pa(e,t,n,s,r){n=n.render;var o=t.ref;return ao(t,r),s=qo(e,t,n,s,o,r),null===e||Ta?(t.effectTag|=1,Oa(e,t,s,r),t.child):(t.updateQueue=e.updateQueue,t.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),qa(e,t,r))}function Fa(e,t,n,s,r,o){if(null===e){var a=n.type;return"function"!=typeof a||Dc(a)||void 0!==a.defaultProps||null!==n.compare||void 0!==n.defaultProps?((e=Oc(n.type,null,s,null,t.mode,o)).ref=t.ref,e.return=t,t.child=e):(t.tag=15,t.type=a,Na(e,t,a,s,r,o))}return a=e.child,r<o&&(r=a.memoizedProps,(n=null!==(n=n.compare)?n:Us)(r,s)&&e.ref===t.ref)?qa(e,t,o):(t.effectTag|=1,(e=Tc(a,s)).ref=t.ref,e.return=t,t.child=e)}function Na(e,t,n,s,r,o){return null!==e&&Us(e.memoizedProps,s)&&e.ref===t.ref&&(Ta=!1,r<o)?(t.expirationTime=e.expirationTime,qa(e,t,o)):Va(e,t,n,s,o)}function Ba(e,t){var n=t.ref;(null===e&&null!==n||null!==e&&e.ref!==n)&&(t.effectTag|=128)}function Va(e,t,n,s,r){var o=xr(n)?br:gr.current;return o=yr(t,o),ao(t,r),n=qo(e,t,n,s,o,r),null===e||Ta?(t.effectTag|=1,Oa(e,t,n,r),t.child):(t.updateQueue=e.updateQueue,t.effectTag&=-517,e.expirationTime<=r&&(e.expirationTime=0),qa(e,t,r))}function Ga(e,t,n,s,r){if(xr(n)){var o=!0;Er(t)}else o=!1;if(ao(t,r),null===t.stateNode)null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),So(t,n,s),Lo(t,n,s,r),s=!0;else if(null===e){var a=t.stateNode,i=t.memoizedProps;a.props=i;var c=a.context,l=n.contextType;"object"==typeof l&&null!==l?l=io(l):l=yr(t,l=xr(n)?br:gr.current);var d=n.getDerivedStateFromProps,p="function"==typeof d||"function"==typeof a.getSnapshotBeforeUpdate;p||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(i!==s||c!==l)&&Eo(t,a,s,l),co=!1;var u=t.memoizedState;a.state=u,go(t,s,a,r),c=t.memoizedState,i!==s||u!==c||fr.current||co?("function"==typeof d&&(xo(t,n,d,s),c=t.memoizedState),(i=co||Ao(t,n,i,s,u,c,l))?(p||"function"!=typeof a.UNSAFE_componentWillMount&&"function"!=typeof a.componentWillMount||("function"==typeof a.componentWillMount&&a.componentWillMount(),"function"==typeof a.UNSAFE_componentWillMount&&a.UNSAFE_componentWillMount()),"function"==typeof a.componentDidMount&&(t.effectTag|=4)):("function"==typeof a.componentDidMount&&(t.effectTag|=4),t.memoizedProps=s,t.memoizedState=c),a.props=s,a.state=c,a.context=l,s=i):("function"==typeof a.componentDidMount&&(t.effectTag|=4),s=!1)}else a=t.stateNode,po(e,t),i=t.memoizedProps,a.props=t.type===t.elementType?i:qr(t.type,i),c=a.context,"object"==typeof(l=n.contextType)&&null!==l?l=io(l):l=yr(t,l=xr(n)?br:gr.current),(p="function"==typeof(d=n.getDerivedStateFromProps)||"function"==typeof a.getSnapshotBeforeUpdate)||"function"!=typeof a.UNSAFE_componentWillReceiveProps&&"function"!=typeof a.componentWillReceiveProps||(i!==s||c!==l)&&Eo(t,a,s,l),co=!1,c=t.memoizedState,a.state=c,go(t,s,a,r),u=t.memoizedState,i!==s||c!==u||fr.current||co?("function"==typeof d&&(xo(t,n,d,s),u=t.memoizedState),(d=co||Ao(t,n,i,s,c,u,l))?(p||"function"!=typeof a.UNSAFE_componentWillUpdate&&"function"!=typeof a.componentWillUpdate||("function"==typeof a.componentWillUpdate&&a.componentWillUpdate(s,u,l),"function"==typeof a.UNSAFE_componentWillUpdate&&a.UNSAFE_componentWillUpdate(s,u,l)),"function"==typeof a.componentDidUpdate&&(t.effectTag|=4),"function"==typeof a.getSnapshotBeforeUpdate&&(t.effectTag|=256)):("function"!=typeof a.componentDidUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=256),t.memoizedProps=s,t.memoizedState=u),a.props=s,a.state=u,a.context=l,s=d):("function"!=typeof a.componentDidUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=4),"function"!=typeof a.getSnapshotBeforeUpdate||i===e.memoizedProps&&c===e.memoizedState||(t.effectTag|=256),s=!1);return Wa(e,t,n,s,o,r)}function Wa(e,t,n,s,r,o){Ba(e,t);var a=0!=(64&t.effectTag);if(!s&&!a)return r&&Lr(t,n,!1),qa(e,t,o);s=t.stateNode,Da.current=t;var i=a&&"function"!=typeof n.getDerivedStateFromError?null:s.render();return t.effectTag|=1,null!==e&&a?(t.child=Io(t,e.child,null,o),t.child=Io(t,null,i,o)):Oa(e,t,i,o),t.memoizedState=s.state,r&&Lr(t,n,!0),t.child}function Ua(e){var t=e.stateNode;t.pendingContext?Ar(0,t.pendingContext,t.pendingContext!==t.context):t.context&&Ar(0,t.context,!1),Po(e,t.containerInfo)}var za,Ha,ja,$a={dehydrated:null,retryTime:0};function Za(e,t,n){var s,r=t.mode,o=t.pendingProps,a=Vo.current,i=!1;if((s=0!=(64&t.effectTag))||(s=0!=(2&a)&&(null===e||null!==e.memoizedState)),s?(i=!0,t.effectTag&=-65):null!==e&&null===e.memoizedState||void 0===o.fallback||!0===o.unstable_avoidThisFallback||(a|=1),mr(Vo,1&a),null===e){if(void 0!==o.fallback&&Ia(t),i){if(i=o.fallback,(o=Pc(null,r,0,null)).return=t,0==(2&t.mode))for(e=null!==t.memoizedState?t.child.child:t.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(n=Pc(i,r,n,null)).return=t,o.sibling=n,t.memoizedState=$a,t.child=o,n}return r=o.children,t.memoizedState=null,t.child=Co(t,null,r,n)}if(null!==e.memoizedState){if(r=(e=e.child).sibling,i){if(o=o.fallback,(n=Tc(e,e.pendingProps)).return=t,0==(2&t.mode)&&(i=null!==t.memoizedState?t.child.child:t.child)!==e.child)for(n.child=i;null!==i;)i.return=n,i=i.sibling;return(r=Tc(r,o)).return=t,n.sibling=r,n.childExpirationTime=0,t.memoizedState=$a,t.child=n,r}return n=Io(t,e.child,o.children,n),t.memoizedState=null,t.child=n}if(e=e.child,i){if(i=o.fallback,(o=Pc(null,r,0,null)).return=t,o.child=e,null!==e&&(e.return=o),0==(2&t.mode))for(e=null!==t.memoizedState?t.child.child:t.child,o.child=e;null!==e;)e.return=o,e=e.sibling;return(n=Pc(i,r,n,null)).return=t,o.sibling=n,n.effectTag|=2,o.childExpirationTime=0,t.memoizedState=$a,t.child=o,n}return t.memoizedState=null,t.child=Io(t,e,o.children,n)}function Xa(e,t){e.expirationTime<t&&(e.expirationTime=t);var n=e.alternate;null!==n&&n.expirationTime<t&&(n.expirationTime=t),oo(e.return,t)}function Ka(e,t,n,s,r,o){var a=e.memoizedState;null===a?e.memoizedState={isBackwards:t,rendering:null,renderingStartTime:0,last:s,tail:n,tailExpiration:0,tailMode:r,lastEffect:o}:(a.isBackwards=t,a.rendering=null,a.renderingStartTime=0,a.last=s,a.tail=n,a.tailExpiration=0,a.tailMode=r,a.lastEffect=o)}function Ya(e,t,n){var s=t.pendingProps,r=s.revealOrder,o=s.tail;if(Oa(e,t,s.children,n),0!=(2&(s=Vo.current)))s=1&s|2,t.effectTag|=64;else{if(null!==e&&0!=(64&e.effectTag))e:for(e=t.child;null!==e;){if(13===e.tag)null!==e.memoizedState&&Xa(e,n);else if(19===e.tag)Xa(e,n);else if(null!==e.child){e.child.return=e,e=e.child;continue}if(e===t)break e;for(;null===e.sibling;){if(null===e.return||e.return===t)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}s&=1}if(mr(Vo,s),0==(2&t.mode))t.memoizedState=null;else switch(r){case"forwards":for(n=t.child,r=null;null!==n;)null!==(e=n.alternate)&&null===Go(e)&&(r=n),n=n.sibling;null===(n=r)?(r=t.child,t.child=null):(r=n.sibling,n.sibling=null),Ka(t,!1,r,n,o,t.lastEffect);break;case"backwards":for(n=null,r=t.child,t.child=null;null!==r;){if(null!==(e=r.alternate)&&null===Go(e)){t.child=r;break}e=r.sibling,r.sibling=n,n=r,r=e}Ka(t,!0,n,null,o,t.lastEffect);break;case"together":Ka(t,!1,null,null,void 0,t.lastEffect);break;default:t.memoizedState=null}return t.child}function qa(e,t,n){null!==e&&(t.dependencies=e.dependencies);var s=t.expirationTime;if(0!==s&&hc(s),t.childExpirationTime<n)return null;if(null!==e&&t.child!==e.child)throw Error(a(153));if(null!==t.child){for(n=Tc(e=t.child,e.pendingProps),t.child=n,n.return=t;null!==e.sibling;)e=e.sibling,(n=n.sibling=Tc(e,e.pendingProps)).return=t;n.sibling=null}return t.child}function Ja(e,t){switch(e.tailMode){case"hidden":t=e.tail;for(var n=null;null!==t;)null!==t.alternate&&(n=t),t=t.sibling;null===n?e.tail=null:n.sibling=null;break;case"collapsed":n=e.tail;for(var s=null;null!==n;)null!==n.alternate&&(s=n),n=n.sibling;null===s?t||null===e.tail?e.tail=null:e.tail.sibling=null:s.sibling=null}}function ei(e,t,n){var s=t.pendingProps;switch(t.tag){case 2:case 16:case 15:case 0:case 11:case 7:case 8:case 12:case 9:case 14:return null;case 1:case 17:return xr(t.type)&&vr(),null;case 3:return Fo(),ur(fr),ur(gr),(n=t.stateNode).pendingContext&&(n.context=n.pendingContext,n.pendingContext=null),null!==e&&null!==e.child||!Qa(t)||(t.effectTag|=4),null;case 5:Bo(t),n=Oo(To.current);var o=t.type;if(null!==e&&null!=t.stateNode)Ha(e,t,o,s,n),e.ref!==t.ref&&(t.effectTag|=128);else{if(!s){if(null===t.stateNode)throw Error(a(166));return null}if(e=Oo(Ro.current),Qa(t)){s=t.stateNode,o=t.type;var i=t.memoizedProps;switch(s[wn]=t,s[In]=i,o){case"iframe":case"object":case"embed":Zt("load",s);break;case"video":case"audio":for(e=0;e<Ye.length;e++)Zt(Ye[e],s);break;case"source":Zt("error",s);break;case"img":case"image":case"link":Zt("error",s),Zt("load",s);break;case"form":Zt("reset",s),Zt("submit",s);break;case"details":Zt("toggle",s);break;case"input":Ee(s,i),Zt("invalid",s),ln(n,"onChange");break;case"select":s._wrapperState={wasMultiple:!!i.multiple},Zt("invalid",s),ln(n,"onChange");break;case"textarea":Qe(s,i),Zt("invalid",s),ln(n,"onChange")}for(var c in on(o,i),e=null,i)if(i.hasOwnProperty(c)){var l=i[c];"children"===c?"string"==typeof l?s.textContent!==l&&(e=["children",l]):"number"==typeof l&&s.textContent!==""+l&&(e=["children",""+l]):L.hasOwnProperty(c)&&null!=l&&ln(n,c)}switch(o){case"input":ve(s),ke(s,i,!0);break;case"textarea":ve(s),De(s);break;case"select":case"option":break;default:"function"==typeof i.onClick&&(s.onclick=dn)}n=e,t.updateQueue=n,null!==n&&(t.effectTag|=4)}else{switch(c=9===n.nodeType?n:n.ownerDocument,e===cn&&(e=Pe(o)),e===cn?"script"===o?((e=c.createElement("div")).innerHTML="<script><\/script>",e=e.removeChild(e.firstChild)):"string"==typeof s.is?e=c.createElement(o,{is:s.is}):(e=c.createElement(o),"select"===o&&(c=e,s.multiple?c.multiple=!0:s.size&&(c.size=s.size))):e=c.createElementNS(e,o),e[wn]=t,e[In]=s,za(e,t),t.stateNode=e,c=an(o,s),o){case"iframe":case"object":case"embed":Zt("load",e),l=s;break;case"video":case"audio":for(l=0;l<Ye.length;l++)Zt(Ye[l],e);l=s;break;case"source":Zt("error",e),l=s;break;case"img":case"image":case"link":Zt("error",e),Zt("load",e),l=s;break;case"form":Zt("reset",e),Zt("submit",e),l=s;break;case"details":Zt("toggle",e),l=s;break;case"input":Ee(e,s),l=Se(e,s),Zt("invalid",e),ln(n,"onChange");break;case"option":l=we(e,s);break;case"select":e._wrapperState={wasMultiple:!!s.multiple},l=r({},s,{value:void 0}),Zt("invalid",e),ln(n,"onChange");break;case"textarea":Qe(e,s),l=Ce(e,s),Zt("invalid",e),ln(n,"onChange");break;default:l=s}on(o,l);var d=l;for(i in d)if(d.hasOwnProperty(i)){var p=d[i];"style"===i?sn(e,p):"dangerouslySetInnerHTML"===i?null!=(p=p?p.__html:void 0)&&Ve(e,p):"children"===i?"string"==typeof p?("textarea"!==o||""!==p)&&Ge(e,p):"number"==typeof p&&Ge(e,""+p):"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&"autoFocus"!==i&&(L.hasOwnProperty(i)?null!=p&&ln(n,i):null!=p&&Y(e,i,p,c))}switch(o){case"input":ve(e),ke(e,s,!1);break;case"textarea":ve(e),De(e);break;case"option":null!=s.value&&e.setAttribute("value",""+ye(s.value));break;case"select":e.multiple=!!s.multiple,null!=(n=s.value)?Ie(e,!!s.multiple,n,!1):null!=s.defaultValue&&Ie(e,!!s.multiple,s.defaultValue,!0);break;default:"function"==typeof l.onClick&&(e.onclick=dn)}An(o,s)&&(t.effectTag|=4)}null!==t.ref&&(t.effectTag|=128)}return null;case 6:if(e&&null!=t.stateNode)ja(0,t,e.memoizedProps,s);else{if("string"!=typeof s&&null===t.stateNode)throw Error(a(166));n=Oo(To.current),Oo(Ro.current),Qa(t)?(n=t.stateNode,s=t.memoizedProps,n[wn]=t,n.nodeValue!==s&&(t.effectTag|=4)):((n=(9===n.nodeType?n:n.ownerDocument).createTextNode(s))[wn]=t,t.stateNode=n)}return null;case 13:return ur(Vo),s=t.memoizedState,0!=(64&t.effectTag)?(t.expirationTime=n,t):(n=null!==s,s=!1,null===e?void 0!==t.memoizedProps.fallback&&Qa(t):(s=null!==(o=e.memoizedState),n||null===o||null!==(o=e.child.sibling)&&(null!==(i=t.firstEffect)?(t.firstEffect=o,o.nextEffect=i):(t.firstEffect=t.lastEffect=o,o.nextEffect=null),o.effectTag=8)),n&&!s&&0!=(2&t.mode)&&(null===e&&!0!==t.memoizedProps.unstable_avoidThisFallback||0!=(1&Vo.current)?Oi===wi&&(Oi=Ii):(Oi!==wi&&Oi!==Ii||(Oi=Ci),0!==Vi&&null!==Ri&&(Gc(Ri,Ti),Wc(Ri,Vi)))),(n||s)&&(t.effectTag|=4),null);case 4:return Fo(),null;case 10:return ro(t),null;case 19:if(ur(Vo),null===(s=t.memoizedState))return null;if(o=0!=(64&t.effectTag),null===(i=s.rendering)){if(o)Ja(s,!1);else if(Oi!==wi||null!==e&&0!=(64&e.effectTag))for(i=t.child;null!==i;){if(null!==(e=Go(i))){for(t.effectTag|=64,Ja(s,!1),null!==(o=e.updateQueue)&&(t.updateQueue=o,t.effectTag|=4),null===s.lastEffect&&(t.firstEffect=null),t.lastEffect=s.lastEffect,s=t.child;null!==s;)i=n,(o=s).effectTag&=2,o.nextEffect=null,o.firstEffect=null,o.lastEffect=null,null===(e=o.alternate)?(o.childExpirationTime=0,o.expirationTime=i,o.child=null,o.memoizedProps=null,o.memoizedState=null,o.updateQueue=null,o.dependencies=null):(o.childExpirationTime=e.childExpirationTime,o.expirationTime=e.expirationTime,o.child=e.child,o.memoizedProps=e.memoizedProps,o.memoizedState=e.memoizedState,o.updateQueue=e.updateQueue,i=e.dependencies,o.dependencies=null===i?null:{expirationTime:i.expirationTime,firstContext:i.firstContext,responders:i.responders}),s=s.sibling;return mr(Vo,1&Vo.current|2),t.child}i=i.sibling}}else{if(!o)if(null!==(e=Go(i))){if(t.effectTag|=64,o=!0,null!==(n=e.updateQueue)&&(t.updateQueue=n,t.effectTag|=4),Ja(s,!0),null===s.tail&&"hidden"===s.tailMode&&!i.alternate)return null!==(t=t.lastEffect=s.lastEffect)&&(t.nextEffect=null),null}else 2*Ur()-s.renderingStartTime>s.tailExpiration&&1<n&&(t.effectTag|=64,o=!0,Ja(s,!1),t.expirationTime=t.childExpirationTime=n-1);s.isBackwards?(i.sibling=t.child,t.child=i):(null!==(n=s.last)?n.sibling=i:t.child=i,s.last=i)}return null!==s.tail?(0===s.tailExpiration&&(s.tailExpiration=Ur()+500),n=s.tail,s.rendering=n,s.tail=n.sibling,s.lastEffect=t.lastEffect,s.renderingStartTime=Ur(),n.sibling=null,t=Vo.current,mr(Vo,o?1&t|2:1&t),n):null}throw Error(a(156,t.tag))}function ti(e){switch(e.tag){case 1:xr(e.type)&&vr();var t=e.effectTag;return 4096&t?(e.effectTag=-4097&t|64,e):null;case 3:if(Fo(),ur(fr),ur(gr),0!=(64&(t=e.effectTag)))throw Error(a(285));return e.effectTag=-4097&t|64,e;case 5:return Bo(e),null;case 13:return ur(Vo),4096&(t=e.effectTag)?(e.effectTag=-4097&t|64,e):null;case 19:return ur(Vo),null;case 4:return Fo(),null;case 10:return ro(e),null;default:return null}}function ni(e,t){return{value:e,source:t,stack:be(t)}}za=function(e,t){for(var n=t.child;null!==n;){if(5===n.tag||6===n.tag)e.appendChild(n.stateNode);else if(4!==n.tag&&null!==n.child){n.child.return=n,n=n.child;continue}if(n===t)break;for(;null===n.sibling;){if(null===n.return||n.return===t)return;n=n.return}n.sibling.return=n.return,n=n.sibling}},Ha=function(e,t,n,s,o){var a=e.memoizedProps;if(a!==s){var i,c,l=t.stateNode;switch(Oo(Ro.current),e=null,n){case"input":a=Se(l,a),s=Se(l,s),e=[];break;case"option":a=we(l,a),s=we(l,s),e=[];break;case"select":a=r({},a,{value:void 0}),s=r({},s,{value:void 0}),e=[];break;case"textarea":a=Ce(l,a),s=Ce(l,s),e=[];break;default:"function"!=typeof a.onClick&&"function"==typeof s.onClick&&(l.onclick=dn)}for(i in on(n,s),n=null,a)if(!s.hasOwnProperty(i)&&a.hasOwnProperty(i)&&null!=a[i])if("style"===i)for(c in l=a[i])l.hasOwnProperty(c)&&(n||(n={}),n[c]="");else"dangerouslySetInnerHTML"!==i&&"children"!==i&&"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&"autoFocus"!==i&&(L.hasOwnProperty(i)?e||(e=[]):(e=e||[]).push(i,null));for(i in s){var d=s[i];if(l=null!=a?a[i]:void 0,s.hasOwnProperty(i)&&d!==l&&(null!=d||null!=l))if("style"===i)if(l){for(c in l)!l.hasOwnProperty(c)||d&&d.hasOwnProperty(c)||(n||(n={}),n[c]="");for(c in d)d.hasOwnProperty(c)&&l[c]!==d[c]&&(n||(n={}),n[c]=d[c])}else n||(e||(e=[]),e.push(i,n)),n=d;else"dangerouslySetInnerHTML"===i?(d=d?d.__html:void 0,l=l?l.__html:void 0,null!=d&&l!==d&&(e=e||[]).push(i,d)):"children"===i?l===d||"string"!=typeof d&&"number"!=typeof d||(e=e||[]).push(i,""+d):"suppressContentEditableWarning"!==i&&"suppressHydrationWarning"!==i&&(L.hasOwnProperty(i)?(null!=d&&ln(o,i),e||l===d||(e=[])):(e=e||[]).push(i,d))}n&&(e=e||[]).push("style",n),o=e,(t.updateQueue=o)&&(t.effectTag|=4)}},ja=function(e,t,n,s){n!==s&&(t.effectTag|=4)};var si="function"==typeof WeakSet?WeakSet:Set;function ri(e,t){var n=t.source,s=t.stack;null===s&&null!==n&&(s=be(n)),null!==n&&fe(n.type),t=t.value,null!==e&&1===e.tag&&fe(e.type);try{console.error(t)}catch(r){setTimeout((function(){throw r}))}}function oi(e){var t=e.ref;if(null!==t)if("function"==typeof t)try{t(null)}catch(n){kc(e,n)}else t.current=null}function ai(e,t){switch(t.tag){case 0:case 11:case 15:case 22:case 3:case 5:case 6:case 4:case 17:return;case 1:if(256&t.effectTag&&null!==e){var n=e.memoizedProps,s=e.memoizedState;t=(e=t.stateNode).getSnapshotBeforeUpdate(t.elementType===t.type?n:qr(t.type,n),s),e.__reactInternalSnapshotBeforeUpdate=t}return}throw Error(a(163))}function ii(e,t){if(null!==(t=null!==(t=t.updateQueue)?t.lastEffect:null)){var n=t=t.next;do{if((n.tag&e)===e){var s=n.destroy;n.destroy=void 0,void 0!==s&&s()}n=n.next}while(n!==t)}}function ci(e,t){if(null!==(t=null!==(t=t.updateQueue)?t.lastEffect:null)){var n=t=t.next;do{if((n.tag&e)===e){var s=n.create;n.destroy=s()}n=n.next}while(n!==t)}}function li(e,t,n){switch(n.tag){case 0:case 11:case 15:case 22:return void ci(3,n);case 1:if(e=n.stateNode,4&n.effectTag)if(null===t)e.componentDidMount();else{var s=n.elementType===n.type?t.memoizedProps:qr(n.type,t.memoizedProps);e.componentDidUpdate(s,t.memoizedState,e.__reactInternalSnapshotBeforeUpdate)}return void(null!==(t=n.updateQueue)&&fo(n,t,e));case 3:if(null!==(t=n.updateQueue)){if(e=null,null!==n.child)switch(n.child.tag){case 5:case 1:e=n.child.stateNode}fo(n,t,e)}return;case 5:return e=n.stateNode,void(null===t&&4&n.effectTag&&An(n.type,n.memoizedProps)&&e.focus());case 6:case 4:case 12:case 19:case 17:case 20:case 21:return;case 13:return void(null===n.memoizedState&&(n=n.alternate,null!==n&&(n=n.memoizedState,null!==n&&(n=n.dehydrated,null!==n&&Ft(n)))))}throw Error(a(163))}function di(e,t,n){switch("function"==typeof Cc&&Cc(t),t.tag){case 0:case 11:case 14:case 15:case 22:if(null!==(e=t.updateQueue)&&null!==(e=e.lastEffect)){var s=e.next;jr(97<n?97:n,(function(){var e=s;do{var n=e.destroy;if(void 0!==n){var r=t;try{n()}catch(o){kc(r,o)}}e=e.next}while(e!==s)}))}break;case 1:oi(t),"function"==typeof(n=t.stateNode).componentWillUnmount&&function(e,t){try{t.props=e.memoizedProps,t.state=e.memoizedState,t.componentWillUnmount()}catch(n){kc(e,n)}}(t,n);break;case 5:oi(t);break;case 4:fi(e,t,n)}}function pi(e){var t=e.alternate;e.return=null,e.child=null,e.memoizedState=null,e.updateQueue=null,e.dependencies=null,e.alternate=null,e.firstEffect=null,e.lastEffect=null,e.pendingProps=null,e.memoizedProps=null,e.stateNode=null,null!==t&&pi(t)}function ui(e){return 5===e.tag||3===e.tag||4===e.tag}function mi(e){e:{for(var t=e.return;null!==t;){if(ui(t)){var n=t;break e}t=t.return}throw Error(a(160))}switch(t=n.stateNode,n.tag){case 5:var s=!1;break;case 3:case 4:t=t.containerInfo,s=!0;break;default:throw Error(a(161))}16&n.effectTag&&(Ge(t,""),n.effectTag&=-17);e:t:for(n=e;;){for(;null===n.sibling;){if(null===n.return||ui(n.return)){n=null;break e}n=n.return}for(n.sibling.return=n.return,n=n.sibling;5!==n.tag&&6!==n.tag&&18!==n.tag;){if(2&n.effectTag)continue t;if(null===n.child||4===n.tag)continue t;n.child.return=n,n=n.child}if(!(2&n.effectTag)){n=n.stateNode;break e}}s?hi(e,n,t):gi(e,n,t)}function hi(e,t,n){var s=e.tag,r=5===s||6===s;if(r)e=r?e.stateNode:e.stateNode.instance,t?8===n.nodeType?n.parentNode.insertBefore(e,t):n.insertBefore(e,t):(8===n.nodeType?(t=n.parentNode).insertBefore(e,n):(t=n).appendChild(e),null!=(n=n._reactRootContainer)||null!==t.onclick||(t.onclick=dn));else if(4!==s&&null!==(e=e.child))for(hi(e,t,n),e=e.sibling;null!==e;)hi(e,t,n),e=e.sibling}function gi(e,t,n){var s=e.tag,r=5===s||6===s;if(r)e=r?e.stateNode:e.stateNode.instance,t?n.insertBefore(e,t):n.appendChild(e);else if(4!==s&&null!==(e=e.child))for(gi(e,t,n),e=e.sibling;null!==e;)gi(e,t,n),e=e.sibling}function fi(e,t,n){for(var s,r,o=t,i=!1;;){if(!i){i=o.return;e:for(;;){if(null===i)throw Error(a(160));switch(s=i.stateNode,i.tag){case 5:r=!1;break e;case 3:case 4:s=s.containerInfo,r=!0;break e}i=i.return}i=!0}if(5===o.tag||6===o.tag){e:for(var c=e,l=o,d=n,p=l;;)if(di(c,p,d),null!==p.child&&4!==p.tag)p.child.return=p,p=p.child;else{if(p===l)break e;for(;null===p.sibling;){if(null===p.return||p.return===l)break e;p=p.return}p.sibling.return=p.return,p=p.sibling}r?(c=s,l=o.stateNode,8===c.nodeType?c.parentNode.removeChild(l):c.removeChild(l)):s.removeChild(o.stateNode)}else if(4===o.tag){if(null!==o.child){s=o.stateNode.containerInfo,r=!0,o.child.return=o,o=o.child;continue}}else if(di(e,o,n),null!==o.child){o.child.return=o,o=o.child;continue}if(o===t)break;for(;null===o.sibling;){if(null===o.return||o.return===t)return;4===(o=o.return).tag&&(i=!1)}o.sibling.return=o.return,o=o.sibling}}function bi(e,t){switch(t.tag){case 0:case 11:case 14:case 15:case 22:return void ii(3,t);case 1:case 12:case 17:return;case 5:var n=t.stateNode;if(null!=n){var s=t.memoizedProps,r=null!==e?e.memoizedProps:s;e=t.type;var o=t.updateQueue;if(t.updateQueue=null,null!==o){for(n[In]=s,"input"===e&&"radio"===s.type&&null!=s.name&&Le(n,s),an(e,r),t=an(e,s),r=0;r<o.length;r+=2){var i=o[r],c=o[r+1];"style"===i?sn(n,c):"dangerouslySetInnerHTML"===i?Ve(n,c):"children"===i?Ge(n,c):Y(n,i,c,t)}switch(e){case"input":Me(n,s);break;case"textarea":Re(n,s);break;case"select":t=n._wrapperState.wasMultiple,n._wrapperState.wasMultiple=!!s.multiple,null!=(e=s.value)?Ie(n,!!s.multiple,e,!1):t!==!!s.multiple&&(null!=s.defaultValue?Ie(n,!!s.multiple,s.defaultValue,!0):Ie(n,!!s.multiple,s.multiple?[]:"",!1))}}}return;case 6:if(null===t.stateNode)throw Error(a(162));return void(t.stateNode.nodeValue=t.memoizedProps);case 3:return void((t=t.stateNode).hydrate&&(t.hydrate=!1,Ft(t.containerInfo)));case 13:if(n=t,null===t.memoizedState?s=!1:(s=!0,n=t.child,Wi=Ur()),null!==n)e:for(e=n;;){if(5===e.tag)o=e.stateNode,s?"function"==typeof(o=o.style).setProperty?o.setProperty("display","none","important"):o.display="none":(o=e.stateNode,r=null!=(r=e.memoizedProps.style)&&r.hasOwnProperty("display")?r.display:null,o.style.display=nn("display",r));else if(6===e.tag)e.stateNode.nodeValue=s?"":e.memoizedProps;else{if(13===e.tag&&null!==e.memoizedState&&null===e.memoizedState.dehydrated){(o=e.child.sibling).return=e,e=o;continue}if(null!==e.child){e.child.return=e,e=e.child;continue}}if(e===n)break;for(;null===e.sibling;){if(null===e.return||e.return===n)break e;e=e.return}e.sibling.return=e.return,e=e.sibling}return void yi(t);case 19:return void yi(t)}throw Error(a(163))}function yi(e){var t=e.updateQueue;if(null!==t){e.updateQueue=null;var n=e.stateNode;null===n&&(n=e.stateNode=new si),t.forEach((function(t){var s=wc.bind(null,e,t);n.has(t)||(n.add(t),t.then(s,s))}))}}var xi="function"==typeof WeakMap?WeakMap:Map;function vi(e,t,n){(n=uo(n,null)).tag=3,n.payload={element:null};var s=t.value;return n.callback=function(){zi||(zi=!0,Hi=s),ri(e,t)},n}function Ai(e,t,n){(n=uo(n,null)).tag=3;var s=e.type.getDerivedStateFromError;if("function"==typeof s){var r=t.value;n.payload=function(){return ri(e,t),s(r)}}var o=e.stateNode;return null!==o&&"function"==typeof o.componentDidCatch&&(n.callback=function(){"function"!=typeof s&&(null===ji?ji=new Set([this]):ji.add(this),ri(e,t));var n=t.stack;this.componentDidCatch(t.value,{componentStack:null!==n?n:""})}),n}var Si,Ei=Math.ceil,Li=K.ReactCurrentDispatcher,Mi=K.ReactCurrentOwner,ki=16,_i=32,wi=0,Ii=3,Ci=4,Qi=0,Ri=null,Di=null,Ti=0,Oi=wi,Pi=null,Fi=1073741823,Ni=1073741823,Bi=null,Vi=0,Gi=!1,Wi=0,Ui=null,zi=!1,Hi=null,ji=null,$i=!1,Zi=null,Xi=90,Ki=null,Yi=0,qi=null,Ji=0;function ec(){return 0!=(48&Qi)?1073741821-(Ur()/10|0):0!==Ji?Ji:Ji=1073741821-(Ur()/10|0)}function tc(e,t,n){if(0==(2&(t=t.mode)))return 1073741823;var s=zr();if(0==(4&t))return 99===s?1073741823:1073741822;if(0!=(Qi&ki))return Ti;if(null!==n)e=Yr(e,0|n.timeoutMs||5e3,250);else switch(s){case 99:e=1073741823;break;case 98:e=Yr(e,150,100);break;case 97:case 96:e=Yr(e,5e3,250);break;case 95:e=2;break;default:throw Error(a(326))}return null!==Ri&&e===Ti&&--e,e}function nc(e,t){if(50<Yi)throw Yi=0,qi=null,Error(a(185));if(null!==(e=sc(e,t))){var n=zr();1073741823===t?0!=(8&Qi)&&0==(48&Qi)?ic(e):(oc(e),0===Qi&&Xr()):oc(e),0==(4&Qi)||98!==n&&99!==n||(null===Ki?Ki=new Map([[e,t]]):(void 0===(n=Ki.get(e))||n>t)&&Ki.set(e,t))}}function sc(e,t){e.expirationTime<t&&(e.expirationTime=t);var n=e.alternate;null!==n&&n.expirationTime<t&&(n.expirationTime=t);var s=e.return,r=null;if(null===s&&3===e.tag)r=e.stateNode;else for(;null!==s;){if(n=s.alternate,s.childExpirationTime<t&&(s.childExpirationTime=t),null!==n&&n.childExpirationTime<t&&(n.childExpirationTime=t),null===s.return&&3===s.tag){r=s.stateNode;break}s=s.return}return null!==r&&(Ri===r&&(hc(t),Oi===Ci&&Gc(r,Ti)),Wc(r,t)),r}function rc(e){var t=e.lastExpiredTime;if(0!==t)return t;if(!Vc(e,t=e.firstPendingTime))return t;var n=e.lastPingedTime;return 2>=(e=n>(e=e.nextKnownPendingLevel)?n:e)&&t!==e?0:e}function oc(e){if(0!==e.lastExpiredTime)e.callbackExpirationTime=1073741823,e.callbackPriority=99,e.callbackNode=Zr(ic.bind(null,e));else{var t=rc(e),n=e.callbackNode;if(0===t)null!==n&&(e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90);else{var s=ec();if(1073741823===t?s=99:1===t||2===t?s=95:s=0>=(s=10*(1073741821-t)-10*(1073741821-s))?99:250>=s?98:5250>=s?97:95,null!==n){var r=e.callbackPriority;if(e.callbackExpirationTime===t&&r>=s)return;n!==Pr&&_r(n)}e.callbackExpirationTime=t,e.callbackPriority=s,t=1073741823===t?Zr(ic.bind(null,e)):$r(s,ac.bind(null,e),{timeout:10*(1073741821-t)-Ur()}),e.callbackNode=t}}}function ac(e,t){if(Ji=0,t)return Uc(e,t=ec()),oc(e),null;var n=rc(e);if(0!==n){if(t=e.callbackNode,0!=(48&Qi))throw Error(a(327));if(Ec(),e===Ri&&n===Ti||dc(e,n),null!==Di){var s=Qi;Qi|=ki;for(var r=uc();;)try{fc();break}catch(c){pc(e,c)}if(so(),Qi=s,Li.current=r,1===Oi)throw t=Pi,dc(e,n),Gc(e,n),oc(e),t;if(null===Di)switch(r=e.finishedWork=e.current.alternate,e.finishedExpirationTime=n,s=Oi,Ri=null,s){case wi:case 1:throw Error(a(345));case 2:Uc(e,2<n?2:n);break;case Ii:if(Gc(e,n),n===(s=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=xc(r)),1073741823===Fi&&10<(r=Wi+500-Ur())){if(Gi){var o=e.lastPingedTime;if(0===o||o>=n){e.lastPingedTime=n,dc(e,n);break}}if(0!==(o=rc(e))&&o!==n)break;if(0!==s&&s!==n){e.lastPingedTime=s;break}e.timeoutHandle=En(vc.bind(null,e),r);break}vc(e);break;case Ci:if(Gc(e,n),n===(s=e.lastSuspendedTime)&&(e.nextKnownPendingLevel=xc(r)),Gi&&(0===(r=e.lastPingedTime)||r>=n)){e.lastPingedTime=n,dc(e,n);break}if(0!==(r=rc(e))&&r!==n)break;if(0!==s&&s!==n){e.lastPingedTime=s;break}if(1073741823!==Ni?s=10*(1073741821-Ni)-Ur():1073741823===Fi?s=0:(s=10*(1073741821-Fi)-5e3,0>(s=(r=Ur())-s)&&(s=0),(n=10*(1073741821-n)-r)<(s=(120>s?120:480>s?480:1080>s?1080:1920>s?1920:3e3>s?3e3:4320>s?4320:1960*Ei(s/1960))-s)&&(s=n)),10<s){e.timeoutHandle=En(vc.bind(null,e),s);break}vc(e);break;case 5:if(1073741823!==Fi&&null!==Bi){o=Fi;var i=Bi;if(0>=(s=0|i.busyMinDurationMs)?s=0:(r=0|i.busyDelayMs,s=(o=Ur()-(10*(1073741821-o)-(0|i.timeoutMs||5e3)))<=r?0:r+s-o),10<s){Gc(e,n),e.timeoutHandle=En(vc.bind(null,e),s);break}}vc(e);break;default:throw Error(a(329))}if(oc(e),e.callbackNode===t)return ac.bind(null,e)}}return null}function ic(e){var t=e.lastExpiredTime;if(t=0!==t?t:1073741823,0!=(48&Qi))throw Error(a(327));if(Ec(),e===Ri&&t===Ti||dc(e,t),null!==Di){var n=Qi;Qi|=ki;for(var s=uc();;)try{gc();break}catch(r){pc(e,r)}if(so(),Qi=n,Li.current=s,1===Oi)throw n=Pi,dc(e,t),Gc(e,t),oc(e),n;if(null!==Di)throw Error(a(261));e.finishedWork=e.current.alternate,e.finishedExpirationTime=t,Ri=null,vc(e),oc(e)}return null}function cc(e,t){var n=Qi;Qi|=1;try{return e(t)}finally{0===(Qi=n)&&Xr()}}function lc(e,t){var n=Qi;Qi&=-2,Qi|=8;try{return e(t)}finally{0===(Qi=n)&&Xr()}}function dc(e,t){e.finishedWork=null,e.finishedExpirationTime=0;var n=e.timeoutHandle;if(-1!==n&&(e.timeoutHandle=-1,Ln(n)),null!==Di)for(n=Di.return;null!==n;){var s=n;switch(s.tag){case 1:null!=(s=s.type.childContextTypes)&&vr();break;case 3:Fo(),ur(fr),ur(gr);break;case 5:Bo(s);break;case 4:Fo();break;case 13:case 19:ur(Vo);break;case 10:ro(s)}n=n.return}Ri=e,Di=Tc(e.current,null),Ti=t,Oi=wi,Pi=null,Ni=Fi=1073741823,Bi=null,Vi=0,Gi=!1}function pc(e,t){for(;;){try{if(so(),Uo.current=va,Xo)for(var n=jo.memoizedState;null!==n;){var s=n.queue;null!==s&&(s.pending=null),n=n.next}if(Ho=0,Zo=$o=jo=null,Xo=!1,null===Di||null===Di.return)return Oi=1,Pi=t,Di=null;e:{var r=e,o=Di.return,a=Di,i=t;if(t=Ti,a.effectTag|=2048,a.firstEffect=a.lastEffect=null,null!==i&&"object"==typeof i&&"function"==typeof i.then){var c=i;if(0==(2&a.mode)){var l=a.alternate;l?(a.updateQueue=l.updateQueue,a.memoizedState=l.memoizedState,a.expirationTime=l.expirationTime):(a.updateQueue=null,a.memoizedState=null)}var d=0!=(1&Vo.current),p=o;do{var u;if(u=13===p.tag){var m=p.memoizedState;if(null!==m)u=null!==m.dehydrated;else{var h=p.memoizedProps;u=void 0!==h.fallback&&(!0!==h.unstable_avoidThisFallback||!d)}}if(u){var g=p.updateQueue;if(null===g){var f=new Set;f.add(c),p.updateQueue=f}else g.add(c);if(0==(2&p.mode)){if(p.effectTag|=64,a.effectTag&=-2981,1===a.tag)if(null===a.alternate)a.tag=17;else{var b=uo(1073741823,null);b.tag=2,mo(a,b)}a.expirationTime=1073741823;break e}i=void 0,a=t;var y=r.pingCache;if(null===y?(y=r.pingCache=new xi,i=new Set,y.set(c,i)):void 0===(i=y.get(c))&&(i=new Set,y.set(c,i)),!i.has(a)){i.add(a);var x=_c.bind(null,r,c,a);c.then(x,x)}p.effectTag|=4096,p.expirationTime=t;break e}p=p.return}while(null!==p);i=Error((fe(a.type)||"A React component")+" suspended while rendering, but no fallback UI was specified.\n\nAdd a <Suspense fallback=...> component higher in the tree to provide a loading indicator or placeholder to display."+be(a))}5!==Oi&&(Oi=2),i=ni(i,a),p=o;do{switch(p.tag){case 3:c=i,p.effectTag|=4096,p.expirationTime=t,ho(p,vi(p,c,t));break e;case 1:c=i;var v=p.type,A=p.stateNode;if(0==(64&p.effectTag)&&("function"==typeof v.getDerivedStateFromError||null!==A&&"function"==typeof A.componentDidCatch&&(null===ji||!ji.has(A)))){p.effectTag|=4096,p.expirationTime=t,ho(p,Ai(p,c,t));break e}}p=p.return}while(null!==p)}Di=yc(Di)}catch(S){t=S;continue}break}}function uc(){var e=Li.current;return Li.current=va,null===e?va:e}function mc(e,t){e<Fi&&2<e&&(Fi=e),null!==t&&e<Ni&&2<e&&(Ni=e,Bi=t)}function hc(e){e>Vi&&(Vi=e)}function gc(){for(;null!==Di;)Di=bc(Di)}function fc(){for(;null!==Di&&!Fr();)Di=bc(Di)}function bc(e){var t=Si(e.alternate,e,Ti);return e.memoizedProps=e.pendingProps,null===t&&(t=yc(e)),Mi.current=null,t}function yc(e){Di=e;do{var t=Di.alternate;if(e=Di.return,0==(2048&Di.effectTag)){if(t=ei(t,Di,Ti),1===Ti||1!==Di.childExpirationTime){for(var n=0,s=Di.child;null!==s;){var r=s.expirationTime,o=s.childExpirationTime;r>n&&(n=r),o>n&&(n=o),s=s.sibling}Di.childExpirationTime=n}if(null!==t)return t;null!==e&&0==(2048&e.effectTag)&&(null===e.firstEffect&&(e.firstEffect=Di.firstEffect),null!==Di.lastEffect&&(null!==e.lastEffect&&(e.lastEffect.nextEffect=Di.firstEffect),e.lastEffect=Di.lastEffect),1<Di.effectTag&&(null!==e.lastEffect?e.lastEffect.nextEffect=Di:e.firstEffect=Di,e.lastEffect=Di))}else{if(null!==(t=ti(Di)))return t.effectTag&=2047,t;null!==e&&(e.firstEffect=e.lastEffect=null,e.effectTag|=2048)}if(null!==(t=Di.sibling))return t;Di=e}while(null!==Di);return Oi===wi&&(Oi=5),null}function xc(e){var t=e.expirationTime;return t>(e=e.childExpirationTime)?t:e}function vc(e){var t=zr();return jr(99,Ac.bind(null,e,t)),null}function Ac(e,t){do{Ec()}while(null!==Zi);if(0!=(48&Qi))throw Error(a(327));var n=e.finishedWork,s=e.finishedExpirationTime;if(null===n)return null;if(e.finishedWork=null,e.finishedExpirationTime=0,n===e.current)throw Error(a(177));e.callbackNode=null,e.callbackExpirationTime=0,e.callbackPriority=90,e.nextKnownPendingLevel=0;var r=xc(n);if(e.firstPendingTime=r,s<=e.lastSuspendedTime?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:s<=e.firstSuspendedTime&&(e.firstSuspendedTime=s-1),s<=e.lastPingedTime&&(e.lastPingedTime=0),s<=e.lastExpiredTime&&(e.lastExpiredTime=0),e===Ri&&(Di=Ri=null,Ti=0),1<n.effectTag?null!==n.lastEffect?(n.lastEffect.nextEffect=n,r=n.firstEffect):r=n:r=n.firstEffect,null!==r){var o=Qi;Qi|=_i,Mi.current=null,xn=$t;var i=gn();if(fn(i)){if("selectionStart"in i)var c={start:i.selectionStart,end:i.selectionEnd};else e:{var l=(c=(c=i.ownerDocument)&&c.defaultView||window).getSelection&&c.getSelection();if(l&&0!==l.rangeCount){c=l.anchorNode;var d=l.anchorOffset,p=l.focusNode;l=l.focusOffset;try{c.nodeType,p.nodeType}catch(k){c=null;break e}var u=0,m=-1,h=-1,g=0,f=0,b=i,y=null;t:for(;;){for(var x;b!==c||0!==d&&3!==b.nodeType||(m=u+d),b!==p||0!==l&&3!==b.nodeType||(h=u+l),3===b.nodeType&&(u+=b.nodeValue.length),null!==(x=b.firstChild);)y=b,b=x;for(;;){if(b===i)break t;if(y===c&&++g===d&&(m=u),y===p&&++f===l&&(h=u),null!==(x=b.nextSibling))break;y=(b=y).parentNode}b=x}c=-1===m||-1===h?null:{start:m,end:h}}else c=null}c=c||{start:0,end:0}}else c=null;vn={activeElementDetached:null,focusedElem:i,selectionRange:c},$t=!1,Ui=r;do{try{Sc()}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);Ui=r;do{try{for(i=e,c=t;null!==Ui;){var v=Ui.effectTag;if(16&v&&Ge(Ui.stateNode,""),128&v){var A=Ui.alternate;if(null!==A){var S=A.ref;null!==S&&("function"==typeof S?S(null):S.current=null)}}switch(1038&v){case 2:mi(Ui),Ui.effectTag&=-3;break;case 6:mi(Ui),Ui.effectTag&=-3,bi(Ui.alternate,Ui);break;case 1024:Ui.effectTag&=-1025;break;case 1028:Ui.effectTag&=-1025,bi(Ui.alternate,Ui);break;case 4:bi(Ui.alternate,Ui);break;case 8:fi(i,d=Ui,c),pi(d)}Ui=Ui.nextEffect}}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);if(S=vn,A=gn(),v=S.focusedElem,c=S.selectionRange,A!==v&&v&&v.ownerDocument&&hn(v.ownerDocument.documentElement,v)){null!==c&&fn(v)&&(A=c.start,void 0===(S=c.end)&&(S=A),"selectionStart"in v?(v.selectionStart=A,v.selectionEnd=Math.min(S,v.value.length)):(S=(A=v.ownerDocument||document)&&A.defaultView||window).getSelection&&(S=S.getSelection(),d=v.textContent.length,i=Math.min(c.start,d),c=void 0===c.end?i:Math.min(c.end,d),!S.extend&&i>c&&(d=c,c=i,i=d),d=mn(v,i),p=mn(v,c),d&&p&&(1!==S.rangeCount||S.anchorNode!==d.node||S.anchorOffset!==d.offset||S.focusNode!==p.node||S.focusOffset!==p.offset)&&((A=A.createRange()).setStart(d.node,d.offset),S.removeAllRanges(),i>c?(S.addRange(A),S.extend(p.node,p.offset)):(A.setEnd(p.node,p.offset),S.addRange(A))))),A=[];for(S=v;S=S.parentNode;)1===S.nodeType&&A.push({element:S,left:S.scrollLeft,top:S.scrollTop});for("function"==typeof v.focus&&v.focus(),v=0;v<A.length;v++)(S=A[v]).element.scrollLeft=S.left,S.element.scrollTop=S.top}$t=!!xn,vn=xn=null,e.current=n,Ui=r;do{try{for(v=e;null!==Ui;){var E=Ui.effectTag;if(36&E&&li(v,Ui.alternate,Ui),128&E){A=void 0;var L=Ui.ref;if(null!==L){var M=Ui.stateNode;Ui.tag,A=M,"function"==typeof L?L(A):L.current=A}}Ui=Ui.nextEffect}}catch(k){if(null===Ui)throw Error(a(330));kc(Ui,k),Ui=Ui.nextEffect}}while(null!==Ui);Ui=null,Nr(),Qi=o}else e.current=n;if($i)$i=!1,Zi=e,Xi=t;else for(Ui=r;null!==Ui;)t=Ui.nextEffect,Ui.nextEffect=null,Ui=t;if(0===(t=e.firstPendingTime)&&(ji=null),1073741823===t?e===qi?Yi++:(Yi=0,qi=e):Yi=0,"function"==typeof Ic&&Ic(n.stateNode,s),oc(e),zi)throw zi=!1,e=Hi,Hi=null,e;return 0!=(8&Qi)||Xr(),null}function Sc(){for(;null!==Ui;){var e=Ui.effectTag;0!=(256&e)&&ai(Ui.alternate,Ui),0==(512&e)||$i||($i=!0,$r(97,(function(){return Ec(),null}))),Ui=Ui.nextEffect}}function Ec(){if(90!==Xi){var e=97<Xi?97:Xi;return Xi=90,jr(e,Lc)}}function Lc(){if(null===Zi)return!1;var e=Zi;if(Zi=null,0!=(48&Qi))throw Error(a(331));var t=Qi;for(Qi|=_i,e=e.current.firstEffect;null!==e;){try{var n=e;if(0!=(512&n.effectTag))switch(n.tag){case 0:case 11:case 15:case 22:ii(5,n),ci(5,n)}}catch(s){if(null===e)throw Error(a(330));kc(e,s)}n=e.nextEffect,e.nextEffect=null,e=n}return Qi=t,Xr(),!0}function Mc(e,t,n){mo(e,t=vi(e,t=ni(n,t),1073741823)),null!==(e=sc(e,1073741823))&&oc(e)}function kc(e,t){if(3===e.tag)Mc(e,e,t);else for(var n=e.return;null!==n;){if(3===n.tag){Mc(n,e,t);break}if(1===n.tag){var s=n.stateNode;if("function"==typeof n.type.getDerivedStateFromError||"function"==typeof s.componentDidCatch&&(null===ji||!ji.has(s))){mo(n,e=Ai(n,e=ni(t,e),1073741823)),null!==(n=sc(n,1073741823))&&oc(n);break}}n=n.return}}function _c(e,t,n){var s=e.pingCache;null!==s&&s.delete(t),Ri===e&&Ti===n?Oi===Ci||Oi===Ii&&1073741823===Fi&&Ur()-Wi<500?dc(e,Ti):Gi=!0:Vc(e,n)&&(0!==(t=e.lastPingedTime)&&t<n||(e.lastPingedTime=n,oc(e)))}function wc(e,t){var n=e.stateNode;null!==n&&n.delete(t),0===(t=0)&&(t=tc(t=ec(),e,null)),null!==(e=sc(e,t))&&oc(e)}Si=function(e,t,n){var s=t.expirationTime;if(null!==e){var r=t.pendingProps;if(e.memoizedProps!==r||fr.current)Ta=!0;else{if(s<n){switch(Ta=!1,t.tag){case 3:Ua(t),Ra();break;case 5:if(No(t),4&t.mode&&1!==n&&r.hidden)return t.expirationTime=t.childExpirationTime=1,null;break;case 1:xr(t.type)&&Er(t);break;case 4:Po(t,t.stateNode.containerInfo);break;case 10:s=t.memoizedProps.value,r=t.type._context,mr(Jr,r._currentValue),r._currentValue=s;break;case 13:if(null!==t.memoizedState)return 0!==(s=t.child.childExpirationTime)&&s>=n?Za(e,t,n):(mr(Vo,1&Vo.current),null!==(t=qa(e,t,n))?t.sibling:null);mr(Vo,1&Vo.current);break;case 19:if(s=t.childExpirationTime>=n,0!=(64&e.effectTag)){if(s)return Ya(e,t,n);t.effectTag|=64}if(null!==(r=t.memoizedState)&&(r.rendering=null,r.tail=null),mr(Vo,Vo.current),!s)return null}return qa(e,t,n)}Ta=!1}}else Ta=!1;switch(t.expirationTime=0,t.tag){case 2:if(s=t.type,null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),e=t.pendingProps,r=yr(t,gr.current),ao(t,n),r=qo(null,t,s,e,r,n),t.effectTag|=1,"object"==typeof r&&null!==r&&"function"==typeof r.render&&void 0===r.$$typeof){if(t.tag=1,t.memoizedState=null,t.updateQueue=null,xr(s)){var o=!0;Er(t)}else o=!1;t.memoizedState=null!==r.state&&void 0!==r.state?r.state:null,lo(t);var i=s.getDerivedStateFromProps;"function"==typeof i&&xo(t,s,i,e),r.updater=vo,t.stateNode=r,r._reactInternalFiber=t,Lo(t,s,e,n),t=Wa(null,t,s,!0,o,n)}else t.tag=0,Oa(null,t,r,n),t=t.child;return t;case 16:e:{if(r=t.elementType,null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),e=t.pendingProps,function(e){if(-1===e._status){e._status=0;var t=e._ctor;t=t(),e._result=t,t.then((function(t){0===e._status&&(t=t.default,e._status=1,e._result=t)}),(function(t){0===e._status&&(e._status=2,e._result=t)}))}}(r),1!==r._status)throw r._result;switch(r=r._result,t.type=r,o=t.tag=function(e){if("function"==typeof e)return Dc(e)?1:0;if(null!=e){if((e=e.$$typeof)===ce)return 11;if(e===pe)return 14}return 2}(r),e=qr(r,e),o){case 0:t=Va(null,t,r,e,n);break e;case 1:t=Ga(null,t,r,e,n);break e;case 11:t=Pa(null,t,r,e,n);break e;case 14:t=Fa(null,t,r,qr(r.type,e),s,n);break e}throw Error(a(306,r,""))}return t;case 0:return s=t.type,r=t.pendingProps,Va(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 1:return s=t.type,r=t.pendingProps,Ga(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 3:if(Ua(t),s=t.updateQueue,null===e||null===s)throw Error(a(282));if(s=t.pendingProps,r=null!==(r=t.memoizedState)?r.element:null,po(e,t),go(t,s,null,n),(s=t.memoizedState.element)===r)Ra(),t=qa(e,t,n);else{if((r=t.stateNode.hydrate)&&(Ma=Mn(t.stateNode.containerInfo.firstChild),La=t,r=ka=!0),r)for(n=Co(t,null,s,n),t.child=n;n;)n.effectTag=-3&n.effectTag|1024,n=n.sibling;else Oa(e,t,s,n),Ra();t=t.child}return t;case 5:return No(t),null===e&&Ia(t),s=t.type,r=t.pendingProps,o=null!==e?e.memoizedProps:null,i=r.children,Sn(s,r)?i=null:null!==o&&Sn(s,o)&&(t.effectTag|=16),Ba(e,t),4&t.mode&&1!==n&&r.hidden?(t.expirationTime=t.childExpirationTime=1,t=null):(Oa(e,t,i,n),t=t.child),t;case 6:return null===e&&Ia(t),null;case 13:return Za(e,t,n);case 4:return Po(t,t.stateNode.containerInfo),s=t.pendingProps,null===e?t.child=Io(t,null,s,n):Oa(e,t,s,n),t.child;case 11:return s=t.type,r=t.pendingProps,Pa(e,t,s,r=t.elementType===s?r:qr(s,r),n);case 7:return Oa(e,t,t.pendingProps,n),t.child;case 8:case 12:return Oa(e,t,t.pendingProps.children,n),t.child;case 10:e:{s=t.type._context,r=t.pendingProps,i=t.memoizedProps,o=r.value;var c=t.type._context;if(mr(Jr,c._currentValue),c._currentValue=o,null!==i)if(c=i.value,0===(o=Gs(c,o)?0:0|("function"==typeof s._calculateChangedBits?s._calculateChangedBits(c,o):1073741823))){if(i.children===r.children&&!fr.current){t=qa(e,t,n);break e}}else for(null!==(c=t.child)&&(c.return=t);null!==c;){var l=c.dependencies;if(null!==l){i=c.child;for(var d=l.firstContext;null!==d;){if(d.context===s&&0!=(d.observedBits&o)){1===c.tag&&((d=uo(n,null)).tag=2,mo(c,d)),c.expirationTime<n&&(c.expirationTime=n),null!==(d=c.alternate)&&d.expirationTime<n&&(d.expirationTime=n),oo(c.return,n),l.expirationTime<n&&(l.expirationTime=n);break}d=d.next}}else i=10===c.tag&&c.type===t.type?null:c.child;if(null!==i)i.return=c;else for(i=c;null!==i;){if(i===t){i=null;break}if(null!==(c=i.sibling)){c.return=i.return,i=c;break}i=i.return}c=i}Oa(e,t,r.children,n),t=t.child}return t;case 9:return r=t.type,s=(o=t.pendingProps).children,ao(t,n),s=s(r=io(r,o.unstable_observedBits)),t.effectTag|=1,Oa(e,t,s,n),t.child;case 14:return o=qr(r=t.type,t.pendingProps),Fa(e,t,r,o=qr(r.type,o),s,n);case 15:return Na(e,t,t.type,t.pendingProps,s,n);case 17:return s=t.type,r=t.pendingProps,r=t.elementType===s?r:qr(s,r),null!==e&&(e.alternate=null,t.alternate=null,t.effectTag|=2),t.tag=1,xr(s)?(e=!0,Er(t)):e=!1,ao(t,n),So(t,s,r),Lo(t,s,r,n),Wa(null,t,s,!0,e,n);case 19:return Ya(e,t,n)}throw Error(a(156,t.tag))};var Ic=null,Cc=null;function Qc(e,t,n,s){this.tag=e,this.key=n,this.sibling=this.child=this.return=this.stateNode=this.type=this.elementType=null,this.index=0,this.ref=null,this.pendingProps=t,this.dependencies=this.memoizedState=this.updateQueue=this.memoizedProps=null,this.mode=s,this.effectTag=0,this.lastEffect=this.firstEffect=this.nextEffect=null,this.childExpirationTime=this.expirationTime=0,this.alternate=null}function Rc(e,t,n,s){return new Qc(e,t,n,s)}function Dc(e){return!(!(e=e.prototype)||!e.isReactComponent)}function Tc(e,t){var n=e.alternate;return null===n?((n=Rc(e.tag,t,e.key,e.mode)).elementType=e.elementType,n.type=e.type,n.stateNode=e.stateNode,n.alternate=e,e.alternate=n):(n.pendingProps=t,n.effectTag=0,n.nextEffect=null,n.firstEffect=null,n.lastEffect=null),n.childExpirationTime=e.childExpirationTime,n.expirationTime=e.expirationTime,n.child=e.child,n.memoizedProps=e.memoizedProps,n.memoizedState=e.memoizedState,n.updateQueue=e.updateQueue,t=e.dependencies,n.dependencies=null===t?null:{expirationTime:t.expirationTime,firstContext:t.firstContext,responders:t.responders},n.sibling=e.sibling,n.index=e.index,n.ref=e.ref,n}function Oc(e,t,n,s,r,o){var i=2;if(s=e,"function"==typeof e)Dc(e)&&(i=1);else if("string"==typeof e)i=5;else e:switch(e){case ne:return Pc(n.children,r,o,t);case ie:i=8,r|=7;break;case se:i=8,r|=1;break;case re:return(e=Rc(12,n,t,8|r)).elementType=re,e.type=re,e.expirationTime=o,e;case le:return(e=Rc(13,n,t,r)).type=le,e.elementType=le,e.expirationTime=o,e;case de:return(e=Rc(19,n,t,r)).elementType=de,e.expirationTime=o,e;default:if("object"==typeof e&&null!==e)switch(e.$$typeof){case oe:i=10;break e;case ae:i=9;break e;case ce:i=11;break e;case pe:i=14;break e;case ue:i=16,s=null;break e;case me:i=22;break e}throw Error(a(130,null==e?e:typeof e,""))}return(t=Rc(i,n,t,r)).elementType=e,t.type=s,t.expirationTime=o,t}function Pc(e,t,n,s){return(e=Rc(7,e,s,t)).expirationTime=n,e}function Fc(e,t,n){return(e=Rc(6,e,null,t)).expirationTime=n,e}function Nc(e,t,n){return(t=Rc(4,null!==e.children?e.children:[],e.key,t)).expirationTime=n,t.stateNode={containerInfo:e.containerInfo,pendingChildren:null,implementation:e.implementation},t}function Bc(e,t,n){this.tag=t,this.current=null,this.containerInfo=e,this.pingCache=this.pendingChildren=null,this.finishedExpirationTime=0,this.finishedWork=null,this.timeoutHandle=-1,this.pendingContext=this.context=null,this.hydrate=n,this.callbackNode=null,this.callbackPriority=90,this.lastExpiredTime=this.lastPingedTime=this.nextKnownPendingLevel=this.lastSuspendedTime=this.firstSuspendedTime=this.firstPendingTime=0}function Vc(e,t){var n=e.firstSuspendedTime;return e=e.lastSuspendedTime,0!==n&&n>=t&&e<=t}function Gc(e,t){var n=e.firstSuspendedTime,s=e.lastSuspendedTime;n<t&&(e.firstSuspendedTime=t),(s>t||0===n)&&(e.lastSuspendedTime=t),t<=e.lastPingedTime&&(e.lastPingedTime=0),t<=e.lastExpiredTime&&(e.lastExpiredTime=0)}function Wc(e,t){t>e.firstPendingTime&&(e.firstPendingTime=t);var n=e.firstSuspendedTime;0!==n&&(t>=n?e.firstSuspendedTime=e.lastSuspendedTime=e.nextKnownPendingLevel=0:t>=e.lastSuspendedTime&&(e.lastSuspendedTime=t+1),t>e.nextKnownPendingLevel&&(e.nextKnownPendingLevel=t))}function Uc(e,t){var n=e.lastExpiredTime;(0===n||n>t)&&(e.lastExpiredTime=t)}function zc(e,t,n,s){var r=t.current,o=ec(),i=bo.suspense;o=tc(o,r,i);e:if(n){t:{if(et(n=n._reactInternalFiber)!==n||1!==n.tag)throw Error(a(170));var c=n;do{switch(c.tag){case 3:c=c.stateNode.context;break t;case 1:if(xr(c.type)){c=c.stateNode.__reactInternalMemoizedMergedChildContext;break t}}c=c.return}while(null!==c);throw Error(a(171))}if(1===n.tag){var l=n.type;if(xr(l)){n=Sr(n,l,c);break e}}n=c}else n=hr;return null===t.context?t.context=n:t.pendingContext=n,(t=uo(o,i)).payload={element:e},null!==(s=void 0===s?null:s)&&(t.callback=s),mo(r,t),nc(r,o),o}function Hc(e){return(e=e.current).child?(e.child.tag,e.child.stateNode):null}function jc(e,t){null!==(e=e.memoizedState)&&null!==e.dehydrated&&e.retryTime<t&&(e.retryTime=t)}function $c(e,t){jc(e,t),(e=e.alternate)&&jc(e,t)}function Zc(e,t,n){var s=new Bc(e,t,n=null!=n&&!0===n.hydrate),r=Rc(3,null,null,2===t?7:1===t?3:0);s.current=r,r.stateNode=s,lo(r),e[Cn]=s.current,n&&0!==t&&function(e,t){var n=Je(t);_t.forEach((function(e){gt(e,t,n)})),wt.forEach((function(e){gt(e,t,n)}))}(0,9===e.nodeType?e:e.ownerDocument),this._internalRoot=s}function Xc(e){return!(!e||1!==e.nodeType&&9!==e.nodeType&&11!==e.nodeType&&(8!==e.nodeType||" react-mount-point-unstable "!==e.nodeValue))}function Kc(e,t,n,s,r){var o=n._reactRootContainer;if(o){var a=o._internalRoot;if("function"==typeof r){var i=r;r=function(){var e=Hc(a);i.call(e)}}zc(t,a,e,r)}else{if(o=n._reactRootContainer=function(e,t){if(t||(t=!(!(t=e?9===e.nodeType?e.documentElement:e.firstChild:null)||1!==t.nodeType||!t.hasAttribute("data-reactroot"))),!t)for(var n;n=e.lastChild;)e.removeChild(n);return new Zc(e,0,t?{hydrate:!0}:void 0)}(n,s),a=o._internalRoot,"function"==typeof r){var c=r;r=function(){var e=Hc(a);c.call(e)}}lc((function(){zc(t,a,e,r)}))}return Hc(a)}function Yc(e,t,n){var s=3<arguments.length&&void 0!==arguments[3]?arguments[3]:null;return{$$typeof:te,key:null==s?null:""+s,children:e,containerInfo:t,implementation:n}}function qc(e,t){var n=2<arguments.length&&void 0!==arguments[2]?arguments[2]:null;if(!Xc(t))throw Error(a(200));return Yc(e,t,null,n)}Zc.prototype.render=function(e){zc(e,this._internalRoot,null,null)},Zc.prototype.unmount=function(){var e=this._internalRoot,t=e.containerInfo;zc(null,e,null,(function(){t[Cn]=null}))},ft=function(e){if(13===e.tag){var t=Yr(ec(),150,100);nc(e,t),$c(e,t)}},bt=function(e){13===e.tag&&(nc(e,3),$c(e,3))},yt=function(e){if(13===e.tag){var t=ec();nc(e,t=tc(t,e,null)),$c(e,t)}},w=function(e,t,n){switch(t){case"input":if(Me(e,n),t=n.name,"radio"===n.type&&null!=t){for(n=e;n.parentNode;)n=n.parentNode;for(n=n.querySelectorAll("input[name="+JSON.stringify(""+t)+'][type="radio"]'),t=0;t<n.length;t++){var s=n[t];if(s!==e&&s.form===e.form){var r=Tn(s);if(!r)throw Error(a(90));Ae(s),Me(s,r)}}}break;case"textarea":Re(e,n);break;case"select":null!=(t=n.value)&&Ie(e,!!n.multiple,t,!1)}},T=cc,O=function(e,t,n,s,r){var o=Qi;Qi|=4;try{return jr(98,e.bind(null,t,n,s,r))}finally{0===(Qi=o)&&Xr()}},P=function(){0==(49&Qi)&&(function(){if(null!==Ki){var e=Ki;Ki=null,e.forEach((function(e,t){Uc(t,e),oc(t)})),Xr()}}(),Ec())},F=function(e,t){var n=Qi;Qi|=2;try{return e(t)}finally{0===(Qi=n)&&Xr()}};var Jc={Events:[Rn,Dn,Tn,k,E,Gn,function(e){ot(e,Vn)},R,D,qt,ct,Ec,{current:!1}]};!function(e){var t=e.findFiberByHostInstance;(function(e){if("undefined"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__)return!1;var t=__REACT_DEVTOOLS_GLOBAL_HOOK__;if(t.isDisabled||!t.supportsFiber)return!0;try{var n=t.inject(e);Ic=function(e){try{t.onCommitFiberRoot(n,e,void 0,64==(64&e.current.effectTag))}catch(s){}},Cc=function(e){try{t.onCommitFiberUnmount(n,e)}catch(s){}}}catch(s){}})(r({},e,{overrideHookState:null,overrideProps:null,setSuspenseHandler:null,scheduleUpdate:null,currentDispatcherRef:K.ReactCurrentDispatcher,findHostInstanceByFiber:function(e){return null===(e=st(e))?null:e.stateNode},findFiberByHostInstance:function(e){return t?t(e):null},findHostInstancesForRefresh:null,scheduleRefresh:null,scheduleRoot:null,setRefreshHandler:null,getCurrentFiber:null}))}({findFiberByHostInstance:Qn,bundleType:0,version:"16.14.0",rendererPackageName:"react-dom"}),t.createPortal=qc,t.hydrate=function(e,t,n){if(!Xc(t))throw Error(a(200));return Kc(null,e,t,!0,n)}},73935:(e,t,n)=>{"use strict";!function e(){if("undefined"!=typeof __REACT_DEVTOOLS_GLOBAL_HOOK__&&"function"==typeof __REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE)try{__REACT_DEVTOOLS_GLOBAL_HOOK__.checkDCE(e)}catch(t){console.error(t)}}(),e.exports=n(64448)},69590:e=>{var t="undefined"!=typeof Element,n="function"==typeof Map,s="function"==typeof Set,r="function"==typeof ArrayBuffer&&!!ArrayBuffer.isView;function o(e,a){if(e===a)return!0;if(e&&a&&"object"==typeof e&&"object"==typeof a){if(e.constructor!==a.constructor)return!1;var i,c,l,d;if(Array.isArray(e)){if((i=e.length)!=a.length)return!1;for(c=i;0!=c--;)if(!o(e[c],a[c]))return!1;return!0}if(n&&e instanceof Map&&a instanceof Map){if(e.size!==a.size)return!1;for(d=e.entries();!(c=d.next()).done;)if(!a.has(c.value[0]))return!1;for(d=e.entries();!(c=d.next()).done;)if(!o(c.value[1],a.get(c.value[0])))return!1;return!0}if(s&&e instanceof Set&&a instanceof Set){if(e.size!==a.size)return!1;for(d=e.entries();!(c=d.next()).done;)if(!a.has(c.value[0]))return!1;return!0}if(r&&ArrayBuffer.isView(e)&&ArrayBuffer.isView(a)){if((i=e.length)!=a.length)return!1;for(c=i;0!=c--;)if(e[c]!==a[c])return!1;return!0}if(e.constructor===RegExp)return e.source===a.source&&e.flags===a.flags;if(e.valueOf!==Object.prototype.valueOf)return e.valueOf()===a.valueOf();if(e.toString!==Object.prototype.toString)return e.toString()===a.toString();if((i=(l=Object.keys(e)).length)!==Object.keys(a).length)return!1;for(c=i;0!=c--;)if(!Object.prototype.hasOwnProperty.call(a,l[c]))return!1;if(t&&e instanceof Element)return!1;for(c=i;0!=c--;)if(("_owner"!==l[c]&&"__v"!==l[c]&&"__o"!==l[c]||!e.$$typeof)&&!o(e[l[c]],a[l[c]]))return!1;return!0}return e!=e&&a!=a}e.exports=function(e,t){try{return o(e,t)}catch(n){if((n.message||"").match(/stack|recursion/i))return console.warn("react-fast-compare cannot handle circular refs"),!1;throw n}}},70405:(e,t,n)=>{"use strict";n.d(t,{B6:()=>H,ql:()=>J});var s=n(67294),r=n(45697),o=n.n(r),a=n(69590),i=n.n(a),c=n(41143),l=n.n(c),d=n(96774),p=n.n(d);function u(){return u=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},u.apply(this,arguments)}function m(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,h(e,t)}function h(e,t){return h=Object.setPrototypeOf||function(e,t){return e.__proto__=t,e},h(e,t)}function g(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)t.indexOf(n=o[s])>=0||(r[n]=e[n]);return r}var f={BASE:"base",BODY:"body",HEAD:"head",HTML:"html",LINK:"link",META:"meta",NOSCRIPT:"noscript",SCRIPT:"script",STYLE:"style",TITLE:"title",FRAGMENT:"Symbol(react.fragment)"},b={rel:["amphtml","canonical","alternate"]},y={type:["application/ld+json"]},x={charset:"",name:["robots","description"],property:["og:type","og:title","og:url","og:image","og:image:alt","og:description","twitter:url","twitter:title","twitter:description","twitter:image","twitter:image:alt","twitter:card","twitter:site"]},v=Object.keys(f).map((function(e){return f[e]})),A={accesskey:"accessKey",charset:"charSet",class:"className",contenteditable:"contentEditable",contextmenu:"contextMenu","http-equiv":"httpEquiv",itemprop:"itemProp",tabindex:"tabIndex"},S=Object.keys(A).reduce((function(e,t){return e[A[t]]=t,e}),{}),E=function(e,t){for(var n=e.length-1;n>=0;n-=1){var s=e[n];if(Object.prototype.hasOwnProperty.call(s,t))return s[t]}return null},L=function(e){var t=E(e,f.TITLE),n=E(e,"titleTemplate");if(Array.isArray(t)&&(t=t.join("")),n&&t)return n.replace(/%s/g,(function(){return t}));var s=E(e,"defaultTitle");return t||s||void 0},M=function(e){return E(e,"onChangeClientState")||function(){}},k=function(e,t){return t.filter((function(t){return void 0!==t[e]})).map((function(t){return t[e]})).reduce((function(e,t){return u({},e,t)}),{})},_=function(e,t){return t.filter((function(e){return void 0!==e[f.BASE]})).map((function(e){return e[f.BASE]})).reverse().reduce((function(t,n){if(!t.length)for(var s=Object.keys(n),r=0;r<s.length;r+=1){var o=s[r].toLowerCase();if(-1!==e.indexOf(o)&&n[o])return t.concat(n)}return t}),[])},w=function(e,t,n){var s={};return n.filter((function(t){return!!Array.isArray(t[e])||(void 0!==t[e]&&console&&"function"==typeof console.warn&&console.warn("Helmet: "+e+' should be of type "Array". Instead found type "'+typeof t[e]+'"'),!1)})).map((function(t){return t[e]})).reverse().reduce((function(e,n){var r={};n.filter((function(e){for(var n,o=Object.keys(e),a=0;a<o.length;a+=1){var i=o[a],c=i.toLowerCase();-1===t.indexOf(c)||"rel"===n&&"canonical"===e[n].toLowerCase()||"rel"===c&&"stylesheet"===e[c].toLowerCase()||(n=c),-1===t.indexOf(i)||"innerHTML"!==i&&"cssText"!==i&&"itemprop"!==i||(n=i)}if(!n||!e[n])return!1;var l=e[n].toLowerCase();return s[n]||(s[n]={}),r[n]||(r[n]={}),!s[n][l]&&(r[n][l]=!0,!0)})).reverse().forEach((function(t){return e.push(t)}));for(var o=Object.keys(r),a=0;a<o.length;a+=1){var i=o[a],c=u({},s[i],r[i]);s[i]=c}return e}),[]).reverse()},I=function(e,t){if(Array.isArray(e)&&e.length)for(var n=0;n<e.length;n+=1)if(e[n][t])return!0;return!1},C=function(e){return Array.isArray(e)?e.join(""):e},Q=function(e,t){return Array.isArray(e)?e.reduce((function(e,n){return function(e,t){for(var n=Object.keys(e),s=0;s<n.length;s+=1)if(t[n[s]]&&t[n[s]].includes(e[n[s]]))return!0;return!1}(n,t)?e.priority.push(n):e.default.push(n),e}),{priority:[],default:[]}):{default:e}},R=function(e,t){var n;return u({},e,((n={})[t]=void 0,n))},D=[f.NOSCRIPT,f.SCRIPT,f.STYLE],T=function(e,t){return void 0===t&&(t=!0),!1===t?String(e):String(e).replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/>/g,"&gt;").replace(/"/g,"&quot;").replace(/'/g,"&#x27;")},O=function(e){return Object.keys(e).reduce((function(t,n){var s=void 0!==e[n]?n+'="'+e[n]+'"':""+n;return t?t+" "+s:s}),"")},P=function(e,t){return void 0===t&&(t={}),Object.keys(e).reduce((function(t,n){return t[A[n]||n]=e[n],t}),t)},F=function(e,t){return t.map((function(t,n){var r,o=((r={key:n})["data-rh"]=!0,r);return Object.keys(t).forEach((function(e){var n=A[e]||e;"innerHTML"===n||"cssText"===n?o.dangerouslySetInnerHTML={__html:t.innerHTML||t.cssText}:o[n]=t[e]})),s.createElement(e,o)}))},N=function(e,t,n){switch(e){case f.TITLE:return{toComponent:function(){return n=t.titleAttributes,(r={key:e=t.title})["data-rh"]=!0,o=P(n,r),[s.createElement(f.TITLE,o,e)];var e,n,r,o},toString:function(){return function(e,t,n,s){var r=O(n),o=C(t);return r?"<"+e+' data-rh="true" '+r+">"+T(o,s)+"</"+e+">":"<"+e+' data-rh="true">'+T(o,s)+"</"+e+">"}(e,t.title,t.titleAttributes,n)}};case"bodyAttributes":case"htmlAttributes":return{toComponent:function(){return P(t)},toString:function(){return O(t)}};default:return{toComponent:function(){return F(e,t)},toString:function(){return function(e,t,n){return t.reduce((function(t,s){var r=Object.keys(s).filter((function(e){return!("innerHTML"===e||"cssText"===e)})).reduce((function(e,t){var r=void 0===s[t]?t:t+'="'+T(s[t],n)+'"';return e?e+" "+r:r}),""),o=s.innerHTML||s.cssText||"",a=-1===D.indexOf(e);return t+"<"+e+' data-rh="true" '+r+(a?"/>":">"+o+"</"+e+">")}),"")}(e,t,n)}}}},B=function(e){var t=e.baseTag,n=e.bodyAttributes,s=e.encode,r=e.htmlAttributes,o=e.noscriptTags,a=e.styleTags,i=e.title,c=void 0===i?"":i,l=e.titleAttributes,d=e.linkTags,p=e.metaTags,u=e.scriptTags,m={toComponent:function(){},toString:function(){return""}};if(e.prioritizeSeoTags){var h=function(e){var t=e.linkTags,n=e.scriptTags,s=e.encode,r=Q(e.metaTags,x),o=Q(t,b),a=Q(n,y);return{priorityMethods:{toComponent:function(){return[].concat(F(f.META,r.priority),F(f.LINK,o.priority),F(f.SCRIPT,a.priority))},toString:function(){return N(f.META,r.priority,s)+" "+N(f.LINK,o.priority,s)+" "+N(f.SCRIPT,a.priority,s)}},metaTags:r.default,linkTags:o.default,scriptTags:a.default}}(e);m=h.priorityMethods,d=h.linkTags,p=h.metaTags,u=h.scriptTags}return{priority:m,base:N(f.BASE,t,s),bodyAttributes:N("bodyAttributes",n,s),htmlAttributes:N("htmlAttributes",r,s),link:N(f.LINK,d,s),meta:N(f.META,p,s),noscript:N(f.NOSCRIPT,o,s),script:N(f.SCRIPT,u,s),style:N(f.STYLE,a,s),title:N(f.TITLE,{title:c,titleAttributes:l},s)}},V=[],G=function(e,t){var n=this;void 0===t&&(t="undefined"!=typeof document),this.instances=[],this.value={setHelmet:function(e){n.context.helmet=e},helmetInstances:{get:function(){return n.canUseDOM?V:n.instances},add:function(e){(n.canUseDOM?V:n.instances).push(e)},remove:function(e){var t=(n.canUseDOM?V:n.instances).indexOf(e);(n.canUseDOM?V:n.instances).splice(t,1)}}},this.context=e,this.canUseDOM=t,t||(e.helmet=B({baseTag:[],bodyAttributes:{},encodeSpecialCharacters:!0,htmlAttributes:{},linkTags:[],metaTags:[],noscriptTags:[],scriptTags:[],styleTags:[],title:"",titleAttributes:{}}))},W=s.createContext({}),U=o().shape({setHelmet:o().func,helmetInstances:o().shape({get:o().func,add:o().func,remove:o().func})}),z="undefined"!=typeof document,H=function(e){function t(n){var s;return(s=e.call(this,n)||this).helmetData=new G(s.props.context,t.canUseDOM),s}return m(t,e),t.prototype.render=function(){return s.createElement(W.Provider,{value:this.helmetData.value},this.props.children)},t}(s.Component);H.canUseDOM=z,H.propTypes={context:o().shape({helmet:o().shape()}),children:o().node.isRequired},H.defaultProps={context:{}},H.displayName="HelmetProvider";var j=function(e,t){var n,s=document.head||document.querySelector(f.HEAD),r=s.querySelectorAll(e+"[data-rh]"),o=[].slice.call(r),a=[];return t&&t.length&&t.forEach((function(t){var s=document.createElement(e);for(var r in t)Object.prototype.hasOwnProperty.call(t,r)&&("innerHTML"===r?s.innerHTML=t.innerHTML:"cssText"===r?s.styleSheet?s.styleSheet.cssText=t.cssText:s.appendChild(document.createTextNode(t.cssText)):s.setAttribute(r,void 0===t[r]?"":t[r]));s.setAttribute("data-rh","true"),o.some((function(e,t){return n=t,s.isEqualNode(e)}))?o.splice(n,1):a.push(s)})),o.forEach((function(e){return e.parentNode.removeChild(e)})),a.forEach((function(e){return s.appendChild(e)})),{oldTags:o,newTags:a}},$=function(e,t){var n=document.getElementsByTagName(e)[0];if(n){for(var s=n.getAttribute("data-rh"),r=s?s.split(","):[],o=[].concat(r),a=Object.keys(t),i=0;i<a.length;i+=1){var c=a[i],l=t[c]||"";n.getAttribute(c)!==l&&n.setAttribute(c,l),-1===r.indexOf(c)&&r.push(c);var d=o.indexOf(c);-1!==d&&o.splice(d,1)}for(var p=o.length-1;p>=0;p-=1)n.removeAttribute(o[p]);r.length===o.length?n.removeAttribute("data-rh"):n.getAttribute("data-rh")!==a.join(",")&&n.setAttribute("data-rh",a.join(","))}},Z=function(e,t){var n=e.baseTag,s=e.htmlAttributes,r=e.linkTags,o=e.metaTags,a=e.noscriptTags,i=e.onChangeClientState,c=e.scriptTags,l=e.styleTags,d=e.title,p=e.titleAttributes;$(f.BODY,e.bodyAttributes),$(f.HTML,s),function(e,t){void 0!==e&&document.title!==e&&(document.title=C(e)),$(f.TITLE,t)}(d,p);var u={baseTag:j(f.BASE,n),linkTags:j(f.LINK,r),metaTags:j(f.META,o),noscriptTags:j(f.NOSCRIPT,a),scriptTags:j(f.SCRIPT,c),styleTags:j(f.STYLE,l)},m={},h={};Object.keys(u).forEach((function(e){var t=u[e],n=t.newTags,s=t.oldTags;n.length&&(m[e]=n),s.length&&(h[e]=u[e].oldTags)})),t&&t(),i(e,m,h)},X=null,K=function(e){function t(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).rendered=!1,t}m(t,e);var n=t.prototype;return n.shouldComponentUpdate=function(e){return!p()(e,this.props)},n.componentDidUpdate=function(){this.emitChange()},n.componentWillUnmount=function(){this.props.context.helmetInstances.remove(this),this.emitChange()},n.emitChange=function(){var e,t,n=this.props.context,s=n.setHelmet,r=null,o=(e=n.helmetInstances.get().map((function(e){var t=u({},e.props);return delete t.context,t})),{baseTag:_(["href"],e),bodyAttributes:k("bodyAttributes",e),defer:E(e,"defer"),encode:E(e,"encodeSpecialCharacters"),htmlAttributes:k("htmlAttributes",e),linkTags:w(f.LINK,["rel","href"],e),metaTags:w(f.META,["name","charset","http-equiv","property","itemprop"],e),noscriptTags:w(f.NOSCRIPT,["innerHTML"],e),onChangeClientState:M(e),scriptTags:w(f.SCRIPT,["src","innerHTML"],e),styleTags:w(f.STYLE,["cssText"],e),title:L(e),titleAttributes:k("titleAttributes",e),prioritizeSeoTags:I(e,"prioritizeSeoTags")});H.canUseDOM?(t=o,X&&cancelAnimationFrame(X),t.defer?X=requestAnimationFrame((function(){Z(t,(function(){X=null}))})):(Z(t),X=null)):B&&(r=B(o)),s(r)},n.init=function(){this.rendered||(this.rendered=!0,this.props.context.helmetInstances.add(this),this.emitChange())},n.render=function(){return this.init(),null},t}(s.Component);K.propTypes={context:U.isRequired},K.displayName="HelmetDispatcher";var Y=["children"],q=["children"],J=function(e){function t(){return e.apply(this,arguments)||this}m(t,e);var n=t.prototype;return n.shouldComponentUpdate=function(e){return!i()(R(this.props,"helmetData"),R(e,"helmetData"))},n.mapNestedChildrenToProps=function(e,t){if(!t)return null;switch(e.type){case f.SCRIPT:case f.NOSCRIPT:return{innerHTML:t};case f.STYLE:return{cssText:t};default:throw new Error("<"+e.type+" /> elements are self-closing and can not contain children. Refer to our API for more information.")}},n.flattenArrayTypeChildren=function(e){var t,n=e.child,s=e.arrayTypeChildren;return u({},s,((t={})[n.type]=[].concat(s[n.type]||[],[u({},e.newChildProps,this.mapNestedChildrenToProps(n,e.nestedChildren))]),t))},n.mapObjectTypeChildren=function(e){var t,n,s=e.child,r=e.newProps,o=e.newChildProps,a=e.nestedChildren;switch(s.type){case f.TITLE:return u({},r,((t={})[s.type]=a,t.titleAttributes=u({},o),t));case f.BODY:return u({},r,{bodyAttributes:u({},o)});case f.HTML:return u({},r,{htmlAttributes:u({},o)});default:return u({},r,((n={})[s.type]=u({},o),n))}},n.mapArrayTypeChildrenToProps=function(e,t){var n=u({},t);return Object.keys(e).forEach((function(t){var s;n=u({},n,((s={})[t]=e[t],s))})),n},n.warnOnInvalidChildren=function(e,t){return l()(v.some((function(t){return e.type===t})),"function"==typeof e.type?"You may be attempting to nest <Helmet> components within each other, which is not allowed. Refer to our API for more information.":"Only elements types "+v.join(", ")+" are allowed. Helmet does not support rendering <"+e.type+"> elements. Refer to our API for more information."),l()(!t||"string"==typeof t||Array.isArray(t)&&!t.some((function(e){return"string"!=typeof e})),"Helmet expects a string as a child of <"+e.type+">. Did you forget to wrap your children in braces? ( <"+e.type+">{``}</"+e.type+"> ) Refer to our API for more information."),!0},n.mapChildrenToProps=function(e,t){var n=this,r={};return s.Children.forEach(e,(function(e){if(e&&e.props){var s=e.props,o=s.children,a=g(s,Y),i=Object.keys(a).reduce((function(e,t){return e[S[t]||t]=a[t],e}),{}),c=e.type;switch("symbol"==typeof c?c=c.toString():n.warnOnInvalidChildren(e,o),c){case f.FRAGMENT:t=n.mapChildrenToProps(o,t);break;case f.LINK:case f.META:case f.NOSCRIPT:case f.SCRIPT:case f.STYLE:r=n.flattenArrayTypeChildren({child:e,arrayTypeChildren:r,newChildProps:i,nestedChildren:o});break;default:t=n.mapObjectTypeChildren({child:e,newProps:t,newChildProps:i,nestedChildren:o})}}})),this.mapArrayTypeChildrenToProps(r,t)},n.render=function(){var e=this.props,t=e.children,n=g(e,q),r=u({},n),o=n.helmetData;return t&&(r=this.mapChildrenToProps(t,r)),!o||o instanceof G||(o=new G(o.context,o.instances)),o?s.createElement(K,u({},r,{context:o.value,helmetData:void 0})):s.createElement(W.Consumer,null,(function(e){return s.createElement(K,u({},r,{context:e}))}))},t}(s.Component);J.propTypes={base:o().object,bodyAttributes:o().object,children:o().oneOfType([o().arrayOf(o().node),o().node]),defaultTitle:o().string,defer:o().bool,encodeSpecialCharacters:o().bool,htmlAttributes:o().object,link:o().arrayOf(o().object),meta:o().arrayOf(o().object),noscript:o().arrayOf(o().object),onChangeClientState:o().func,script:o().arrayOf(o().object),style:o().arrayOf(o().object),title:o().string,titleAttributes:o().object,titleTemplate:o().string,prioritizeSeoTags:o().bool,helmetData:o().object},J.defaultProps={defer:!0,encodeSpecialCharacters:!0,prioritizeSeoTags:!1},J.displayName="Helmet"},69921:(e,t)=>{"use strict";var n="function"==typeof Symbol&&Symbol.for,s=n?Symbol.for("react.element"):60103,r=n?Symbol.for("react.portal"):60106,o=n?Symbol.for("react.fragment"):60107,a=n?Symbol.for("react.strict_mode"):60108,i=n?Symbol.for("react.profiler"):60114,c=n?Symbol.for("react.provider"):60109,l=n?Symbol.for("react.context"):60110,d=n?Symbol.for("react.async_mode"):60111,p=n?Symbol.for("react.concurrent_mode"):60111,u=n?Symbol.for("react.forward_ref"):60112,m=n?Symbol.for("react.suspense"):60113,h=n?Symbol.for("react.suspense_list"):60120,g=n?Symbol.for("react.memo"):60115,f=n?Symbol.for("react.lazy"):60116,b=n?Symbol.for("react.block"):60121,y=n?Symbol.for("react.fundamental"):60117,x=n?Symbol.for("react.responder"):60118,v=n?Symbol.for("react.scope"):60119;function A(e){if("object"==typeof e&&null!==e){var t=e.$$typeof;switch(t){case s:switch(e=e.type){case d:case p:case o:case i:case a:case m:return e;default:switch(e=e&&e.$$typeof){case l:case u:case f:case g:case c:return e;default:return t}}case r:return t}}}function S(e){return A(e)===p}t.AsyncMode=d,t.ConcurrentMode=p,t.ContextConsumer=l,t.ContextProvider=c,t.Element=s,t.ForwardRef=u,t.Fragment=o,t.Lazy=f,t.Memo=g,t.Portal=r,t.Profiler=i,t.StrictMode=a,t.Suspense=m,t.isAsyncMode=function(e){return S(e)||A(e)===d},t.isConcurrentMode=S,t.isContextConsumer=function(e){return A(e)===l},t.isContextProvider=function(e){return A(e)===c},t.isElement=function(e){return"object"==typeof e&&null!==e&&e.$$typeof===s},t.isForwardRef=function(e){return A(e)===u},t.isFragment=function(e){return A(e)===o},t.isLazy=function(e){return A(e)===f},t.isMemo=function(e){return A(e)===g},t.isPortal=function(e){return A(e)===r},t.isProfiler=function(e){return A(e)===i},t.isStrictMode=function(e){return A(e)===a},t.isSuspense=function(e){return A(e)===m},t.isValidElementType=function(e){return"string"==typeof e||"function"==typeof e||e===o||e===p||e===i||e===a||e===m||e===h||"object"==typeof e&&null!==e&&(e.$$typeof===f||e.$$typeof===g||e.$$typeof===c||e.$$typeof===l||e.$$typeof===u||e.$$typeof===y||e.$$typeof===x||e.$$typeof===v||e.$$typeof===b)},t.typeOf=A},59864:(e,t,n)=>{"use strict";e.exports=n(69921)},68356:(e,t,n)=>{"use strict";function s(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,e.__proto__=t}function r(e){if(void 0===e)throw new ReferenceError("this hasn't been initialised - super() hasn't been called");return e}function o(e,t,n){return t in e?Object.defineProperty(e,t,{value:n,enumerable:!0,configurable:!0,writable:!0}):e[t]=n,e}function a(){return a=Object.assign||function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},a.apply(this,arguments)}var i=n(67294),c=n(45697),l=[],d=[];function p(e){var t=e(),n={loading:!0,loaded:null,error:null};return n.promise=t.then((function(e){return n.loading=!1,n.loaded=e,e})).catch((function(e){throw n.loading=!1,n.error=e,e})),n}function u(e){var t={loading:!1,loaded:{},error:null},n=[];try{Object.keys(e).forEach((function(s){var r=p(e[s]);r.loading?t.loading=!0:(t.loaded[s]=r.loaded,t.error=r.error),n.push(r.promise),r.promise.then((function(e){t.loaded[s]=e})).catch((function(e){t.error=e}))}))}catch(s){t.error=s}return t.promise=Promise.all(n).then((function(e){return t.loading=!1,e})).catch((function(e){throw t.loading=!1,e})),t}function m(e,t){return i.createElement((n=e)&&n.__esModule?n.default:n,t);var n}function h(e,t){var p,u;if(!t.loading)throw new Error("react-loadable requires a `loading` component");var h=a({loader:null,loading:null,delay:200,timeout:null,render:m,webpack:null,modules:null},t),g=null;function f(){return g||(g=e(h.loader)),g.promise}return l.push(f),"function"==typeof h.webpack&&d.push((function(){if((0,h.webpack)().every((function(e){return void 0!==e&&void 0!==n.m[e]})))return f()})),u=p=function(t){function n(n){var s;return o(r(r(s=t.call(this,n)||this)),"retry",(function(){s.setState({error:null,loading:!0,timedOut:!1}),g=e(h.loader),s._loadModule()})),f(),s.state={error:g.error,pastDelay:!1,timedOut:!1,loading:g.loading,loaded:g.loaded},s}s(n,t),n.preload=function(){return f()};var a=n.prototype;return a.UNSAFE_componentWillMount=function(){this._loadModule()},a.componentDidMount=function(){this._mounted=!0},a._loadModule=function(){var e=this;if(this.context.loadable&&Array.isArray(h.modules)&&h.modules.forEach((function(t){e.context.loadable.report(t)})),g.loading){var t=function(t){e._mounted&&e.setState(t)};"number"==typeof h.delay&&(0===h.delay?this.setState({pastDelay:!0}):this._delay=setTimeout((function(){t({pastDelay:!0})}),h.delay)),"number"==typeof h.timeout&&(this._timeout=setTimeout((function(){t({timedOut:!0})}),h.timeout));var n=function(){t({error:g.error,loaded:g.loaded,loading:g.loading}),e._clearTimeouts()};g.promise.then((function(){return n(),null})).catch((function(e){return n(),null}))}},a.componentWillUnmount=function(){this._mounted=!1,this._clearTimeouts()},a._clearTimeouts=function(){clearTimeout(this._delay),clearTimeout(this._timeout)},a.render=function(){return this.state.loading||this.state.error?i.createElement(h.loading,{isLoading:this.state.loading,pastDelay:this.state.pastDelay,timedOut:this.state.timedOut,error:this.state.error,retry:this.retry}):this.state.loaded?h.render(this.state.loaded,this.props):null},n}(i.Component),o(p,"contextTypes",{loadable:c.shape({report:c.func.isRequired})}),u}function g(e){return h(p,e)}g.Map=function(e){if("function"!=typeof e.render)throw new Error("LoadableMap requires a `render(loaded, props)` function");return h(u,e)};var f=function(e){function t(){return e.apply(this,arguments)||this}s(t,e);var n=t.prototype;return n.getChildContext=function(){return{loadable:{report:this.props.report}}},n.render=function(){return i.Children.only(this.props.children)},t}(i.Component);function b(e){for(var t=[];e.length;){var n=e.pop();t.push(n())}return Promise.all(t).then((function(){if(e.length)return b(e)}))}o(f,"propTypes",{report:c.func.isRequired}),o(f,"childContextTypes",{loadable:c.shape({report:c.func.isRequired}).isRequired}),g.Capture=f,g.preloadAll=function(){return new Promise((function(e,t){b(l).then(e,t)}))},g.preloadReady=function(){return new Promise((function(e,t){b(d).then(e,e)}))},e.exports=g},18790:(e,t,n)=>{"use strict";n.d(t,{H:()=>i,f:()=>a});var s=n(16550),r=n(83117),o=n(67294);function a(e,t,n){return void 0===n&&(n=[]),e.some((function(e){var r=e.path?(0,s.LX)(t,e):n.length?n[n.length-1].match:s.F0.computeRootMatch(t);return r&&(n.push({route:e,match:r}),e.routes&&a(e.routes,t,n)),r})),n}function i(e,t,n){return void 0===t&&(t={}),void 0===n&&(n={}),e?o.createElement(s.rs,n,e.map((function(e,n){return o.createElement(s.AW,{key:e.key||n,path:e.path,exact:e.exact,strict:e.strict,render:function(n){return e.render?e.render((0,r.Z)({},n,{},t,{route:e})):o.createElement(e.component,(0,r.Z)({},n,t,{route:e}))}})}))):null}},73727:(e,t,n)=>{"use strict";n.d(t,{OL:()=>x,VK:()=>d,rU:()=>f});var s=n(16550),r=n(90144),o=n(67294),a=n(99318),i=n(83117),c=n(80102),l=n(2177),d=function(e){function t(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).history=(0,a.lX)(t.props),t}return(0,r.Z)(t,e),t.prototype.render=function(){return o.createElement(s.F0,{history:this.history,children:this.props.children})},t}(o.Component);o.Component;var p=function(e,t){return"function"==typeof e?e(t):e},u=function(e,t){return"string"==typeof e?(0,a.ob)(e,null,null,t):e},m=function(e){return e},h=o.forwardRef;void 0===h&&(h=m);var g=h((function(e,t){var n=e.innerRef,s=e.navigate,r=e.onClick,a=(0,c.Z)(e,["innerRef","navigate","onClick"]),l=a.target,d=(0,i.Z)({},a,{onClick:function(e){try{r&&r(e)}catch(t){throw e.preventDefault(),t}e.defaultPrevented||0!==e.button||l&&"_self"!==l||function(e){return!!(e.metaKey||e.altKey||e.ctrlKey||e.shiftKey)}(e)||(e.preventDefault(),s())}});return d.ref=m!==h&&t||n,o.createElement("a",d)}));var f=h((function(e,t){var n=e.component,r=void 0===n?g:n,d=e.replace,f=e.to,b=e.innerRef,y=(0,c.Z)(e,["component","replace","to","innerRef"]);return o.createElement(s.s6.Consumer,null,(function(e){e||(0,l.Z)(!1);var n=e.history,s=u(p(f,e.location),e.location),c=s?n.createHref(s):"",g=(0,i.Z)({},y,{href:c,navigate:function(){var t=p(f,e.location),s=(0,a.Ep)(e.location)===(0,a.Ep)(u(t));(d||s?n.replace:n.push)(t)}});return m!==h?g.ref=t||b:g.innerRef=b,o.createElement(r,g)}))})),b=function(e){return e},y=o.forwardRef;void 0===y&&(y=b);var x=y((function(e,t){var n=e["aria-current"],r=void 0===n?"page":n,a=e.activeClassName,d=void 0===a?"active":a,m=e.activeStyle,h=e.className,g=e.exact,x=e.isActive,v=e.location,A=e.sensitive,S=e.strict,E=e.style,L=e.to,M=e.innerRef,k=(0,c.Z)(e,["aria-current","activeClassName","activeStyle","className","exact","isActive","location","sensitive","strict","style","to","innerRef"]);return o.createElement(s.s6.Consumer,null,(function(e){e||(0,l.Z)(!1);var n=v||e.location,a=u(p(L,n),n),c=a.pathname,_=c&&c.replace(/([.+*?=^!:${}()[\]|/\\])/g,"\\$1"),w=_?(0,s.LX)(n.pathname,{path:_,exact:g,sensitive:A,strict:S}):null,I=!!(x?x(w,n):w),C="function"==typeof h?h(I):h,Q="function"==typeof E?E(I):E;I&&(C=function(){for(var e=arguments.length,t=new Array(e),n=0;n<e;n++)t[n]=arguments[n];return t.filter((function(e){return e})).join(" ")}(C,d),Q=(0,i.Z)({},Q,m));var R=(0,i.Z)({"aria-current":I&&r||null,className:C,style:Q,to:a},k);return b!==y?R.ref=t||M:R.innerRef=M,o.createElement(f,R)}))}))},16550:(e,t,n)=>{"use strict";n.d(t,{AW:()=>L,F0:()=>v,LX:()=>E,TH:()=>D,k6:()=>R,rs:()=>C,s6:()=>x});var s=n(90144),r=n(67294),o=n(45697),a=n.n(o),i=n(99318),c=n(2177),l=n(83117),d=n(39658),p=n.n(d),u=(n(59864),n(80102)),m=(n(8679),1073741823),h="undefined"!=typeof globalThis?globalThis:"undefined"!=typeof window?window:void 0!==n.g?n.g:{};function g(e){var t=[];return{on:function(e){t.push(e)},off:function(e){t=t.filter((function(t){return t!==e}))},get:function(){return e},set:function(n,s){e=n,t.forEach((function(t){return t(e,s)}))}}}var f=r.createContext||function(e,t){var n,o,i="__create-react-context-"+function(){var e="__global_unique_id__";return h[e]=(h[e]||0)+1}()+"__",c=function(e){function n(){for(var t,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(t=e.call.apply(e,[this].concat(s))||this).emitter=g(t.props.value),t}(0,s.Z)(n,e);var r=n.prototype;return r.getChildContext=function(){var e;return(e={})[i]=this.emitter,e},r.componentWillReceiveProps=function(e){if(this.props.value!==e.value){var n,s=this.props.value,r=e.value;((o=s)===(a=r)?0!==o||1/o==1/a:o!=o&&a!=a)?n=0:(n="function"==typeof t?t(s,r):m,0!==(n|=0)&&this.emitter.set(e.value,n))}var o,a},r.render=function(){return this.props.children},n}(r.Component);c.childContextTypes=((n={})[i]=a().object.isRequired,n);var l=function(t){function n(){for(var e,n=arguments.length,s=new Array(n),r=0;r<n;r++)s[r]=arguments[r];return(e=t.call.apply(t,[this].concat(s))||this).observedBits=void 0,e.state={value:e.getValue()},e.onUpdate=function(t,n){0!=((0|e.observedBits)&n)&&e.setState({value:e.getValue()})},e}(0,s.Z)(n,t);var r=n.prototype;return r.componentWillReceiveProps=function(e){var t=e.observedBits;this.observedBits=null==t?m:t},r.componentDidMount=function(){this.context[i]&&this.context[i].on(this.onUpdate);var e=this.props.observedBits;this.observedBits=null==e?m:e},r.componentWillUnmount=function(){this.context[i]&&this.context[i].off(this.onUpdate)},r.getValue=function(){return this.context[i]?this.context[i].get():e},r.render=function(){return(e=this.props.children,Array.isArray(e)?e[0]:e)(this.state.value);var e},n}(r.Component);return l.contextTypes=((o={})[i]=a().object,o),{Provider:c,Consumer:l}},b=function(e){var t=f();return t.displayName=e,t},y=b("Router-History"),x=b("Router"),v=function(e){function t(t){var n;return(n=e.call(this,t)||this).state={location:t.history.location},n._isMounted=!1,n._pendingLocation=null,t.staticContext||(n.unlisten=t.history.listen((function(e){n._pendingLocation=e}))),n}(0,s.Z)(t,e),t.computeRootMatch=function(e){return{path:"/",url:"/",params:{},isExact:"/"===e}};var n=t.prototype;return n.componentDidMount=function(){var e=this;this._isMounted=!0,this.unlisten&&this.unlisten(),this.props.staticContext||(this.unlisten=this.props.history.listen((function(t){e._isMounted&&e.setState({location:t})}))),this._pendingLocation&&this.setState({location:this._pendingLocation})},n.componentWillUnmount=function(){this.unlisten&&(this.unlisten(),this._isMounted=!1,this._pendingLocation=null)},n.render=function(){return r.createElement(x.Provider,{value:{history:this.props.history,location:this.state.location,match:t.computeRootMatch(this.state.location.pathname),staticContext:this.props.staticContext}},r.createElement(y.Provider,{children:this.props.children||null,value:this.props.history}))},t}(r.Component);r.Component;r.Component;var A={},S=0;function E(e,t){void 0===t&&(t={}),("string"==typeof t||Array.isArray(t))&&(t={path:t});var n=t,s=n.path,r=n.exact,o=void 0!==r&&r,a=n.strict,i=void 0!==a&&a,c=n.sensitive,l=void 0!==c&&c;return[].concat(s).reduce((function(t,n){if(!n&&""!==n)return null;if(t)return t;var s=function(e,t){var n=""+t.end+t.strict+t.sensitive,s=A[n]||(A[n]={});if(s[e])return s[e];var r=[],o={regexp:p()(e,r,t),keys:r};return S<1e4&&(s[e]=o,S++),o}(n,{end:o,strict:i,sensitive:l}),r=s.regexp,a=s.keys,c=r.exec(e);if(!c)return null;var d=c[0],u=c.slice(1),m=e===d;return o&&!m?null:{path:n,url:"/"===n&&""===d?"/":d,isExact:m,params:a.reduce((function(e,t,n){return e[t.name]=u[n],e}),{})}}),null)}var L=function(e){function t(){return e.apply(this,arguments)||this}return(0,s.Z)(t,e),t.prototype.render=function(){var e=this;return r.createElement(x.Consumer,null,(function(t){t||(0,c.Z)(!1);var n=e.props.location||t.location,s=e.props.computedMatch?e.props.computedMatch:e.props.path?E(n.pathname,e.props):t.match,o=(0,l.Z)({},t,{location:n,match:s}),a=e.props,i=a.children,d=a.component,p=a.render;return Array.isArray(i)&&function(e){return 0===r.Children.count(e)}(i)&&(i=null),r.createElement(x.Provider,{value:o},o.match?i?"function"==typeof i?i(o):i:d?r.createElement(d,o):p?p(o):null:"function"==typeof i?i(o):null)}))},t}(r.Component);function M(e){return"/"===e.charAt(0)?e:"/"+e}function k(e,t){if(!e)return t;var n=M(e);return 0!==t.pathname.indexOf(n)?t:(0,l.Z)({},t,{pathname:t.pathname.substr(n.length)})}function _(e){return"string"==typeof e?e:(0,i.Ep)(e)}function w(e){return function(){(0,c.Z)(!1)}}function I(){}r.Component;var C=function(e){function t(){return e.apply(this,arguments)||this}return(0,s.Z)(t,e),t.prototype.render=function(){var e=this;return r.createElement(x.Consumer,null,(function(t){t||(0,c.Z)(!1);var n,s,o=e.props.location||t.location;return r.Children.forEach(e.props.children,(function(e){if(null==s&&r.isValidElement(e)){n=e;var a=e.props.path||e.props.from;s=a?E(o.pathname,(0,l.Z)({},e.props,{path:a})):t.match}})),s?r.cloneElement(n,{location:o,computedMatch:s}):null}))},t}(r.Component);var Q=r.useContext;function R(){return Q(y)}function D(){return Q(x).location}},39658:(e,t,n)=>{var s=n(5826);e.exports=m,e.exports.parse=o,e.exports.compile=function(e,t){return i(o(e,t),t)},e.exports.tokensToFunction=i,e.exports.tokensToRegExp=u;var r=new RegExp(["(\\\\.)","([\\/.])?(?:(?:\\:(\\w+)(?:\\(((?:\\\\.|[^\\\\()])+)\\))?|\\(((?:\\\\.|[^\\\\()])+)\\))([+*?])?|(\\*))"].join("|"),"g");function o(e,t){for(var n,s=[],o=0,a=0,i="",d=t&&t.delimiter||"/";null!=(n=r.exec(e));){var p=n[0],u=n[1],m=n.index;if(i+=e.slice(a,m),a=m+p.length,u)i+=u[1];else{var h=e[a],g=n[2],f=n[3],b=n[4],y=n[5],x=n[6],v=n[7];i&&(s.push(i),i="");var A=null!=g&&null!=h&&h!==g,S="+"===x||"*"===x,E="?"===x||"*"===x,L=n[2]||d,M=b||y;s.push({name:f||o++,prefix:g||"",delimiter:L,optional:E,repeat:S,partial:A,asterisk:!!v,pattern:M?l(M):v?".*":"[^"+c(L)+"]+?"})}}return a<e.length&&(i+=e.substr(a)),i&&s.push(i),s}function a(e){return encodeURI(e).replace(/[\/?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()}))}function i(e,t){for(var n=new Array(e.length),r=0;r<e.length;r++)"object"==typeof e[r]&&(n[r]=new RegExp("^(?:"+e[r].pattern+")$",p(t)));return function(t,r){for(var o="",i=t||{},c=(r||{}).pretty?a:encodeURIComponent,l=0;l<e.length;l++){var d=e[l];if("string"!=typeof d){var p,u=i[d.name];if(null==u){if(d.optional){d.partial&&(o+=d.prefix);continue}throw new TypeError('Expected "'+d.name+'" to be defined')}if(s(u)){if(!d.repeat)throw new TypeError('Expected "'+d.name+'" to not repeat, but received `'+JSON.stringify(u)+"`");if(0===u.length){if(d.optional)continue;throw new TypeError('Expected "'+d.name+'" to not be empty')}for(var m=0;m<u.length;m++){if(p=c(u[m]),!n[l].test(p))throw new TypeError('Expected all "'+d.name+'" to match "'+d.pattern+'", but received `'+JSON.stringify(p)+"`");o+=(0===m?d.prefix:d.delimiter)+p}}else{if(p=d.asterisk?encodeURI(u).replace(/[?#]/g,(function(e){return"%"+e.charCodeAt(0).toString(16).toUpperCase()})):c(u),!n[l].test(p))throw new TypeError('Expected "'+d.name+'" to match "'+d.pattern+'", but received "'+p+'"');o+=d.prefix+p}}else o+=d}return o}}function c(e){return e.replace(/([.+*?=^!:${}()[\]|\/\\])/g,"\\$1")}function l(e){return e.replace(/([=!:$\/()])/g,"\\$1")}function d(e,t){return e.keys=t,e}function p(e){return e&&e.sensitive?"":"i"}function u(e,t,n){s(t)||(n=t||n,t=[]);for(var r=(n=n||{}).strict,o=!1!==n.end,a="",i=0;i<e.length;i++){var l=e[i];if("string"==typeof l)a+=c(l);else{var u=c(l.prefix),m="(?:"+l.pattern+")";t.push(l),l.repeat&&(m+="(?:"+u+m+")*"),a+=m=l.optional?l.partial?u+"("+m+")?":"(?:"+u+"("+m+"))?":u+"("+m+")"}}var h=c(n.delimiter||"/"),g=a.slice(-h.length)===h;return r||(a=(g?a.slice(0,-h.length):a)+"(?:"+h+"(?=$))?"),a+=o?"$":r&&g?"":"(?="+h+"|$)",d(new RegExp("^"+a,p(n)),t)}function m(e,t,n){return s(t)||(n=t||n,t=[]),n=n||{},e instanceof RegExp?function(e,t){var n=e.source.match(/\((?!\?)/g);if(n)for(var s=0;s<n.length;s++)t.push({name:s,prefix:null,delimiter:null,optional:!1,repeat:!1,partial:!1,asterisk:!1,pattern:null});return d(e,t)}(e,t):s(e)?function(e,t,n){for(var s=[],r=0;r<e.length;r++)s.push(m(e[r],t,n).source);return d(new RegExp("(?:"+s.join("|")+")",p(n)),t)}(e,t,n):function(e,t,n){return u(o(e,n),t,n)}(e,t,n)}},72408:(e,t,n)=>{"use strict";var s=n(27418),r="function"==typeof Symbol&&Symbol.for,o=r?Symbol.for("react.element"):60103,a=r?Symbol.for("react.portal"):60106,i=r?Symbol.for("react.fragment"):60107,c=r?Symbol.for("react.strict_mode"):60108,l=r?Symbol.for("react.profiler"):60114,d=r?Symbol.for("react.provider"):60109,p=r?Symbol.for("react.context"):60110,u=r?Symbol.for("react.forward_ref"):60112,m=r?Symbol.for("react.suspense"):60113,h=r?Symbol.for("react.memo"):60115,g=r?Symbol.for("react.lazy"):60116,f="function"==typeof Symbol&&Symbol.iterator;function b(e){for(var t="https://reactjs.org/docs/error-decoder.html?invariant="+e,n=1;n<arguments.length;n++)t+="&args[]="+encodeURIComponent(arguments[n]);return"Minified React error #"+e+"; visit "+t+" for the full message or use the non-minified dev environment for full errors and additional helpful warnings."}var y={isMounted:function(){return!1},enqueueForceUpdate:function(){},enqueueReplaceState:function(){},enqueueSetState:function(){}},x={};function v(e,t,n){this.props=e,this.context=t,this.refs=x,this.updater=n||y}function A(){}function S(e,t,n){this.props=e,this.context=t,this.refs=x,this.updater=n||y}v.prototype.isReactComponent={},v.prototype.setState=function(e,t){if("object"!=typeof e&&"function"!=typeof e&&null!=e)throw Error(b(85));this.updater.enqueueSetState(this,e,t,"setState")},v.prototype.forceUpdate=function(e){this.updater.enqueueForceUpdate(this,e,"forceUpdate")},A.prototype=v.prototype;var E=S.prototype=new A;E.constructor=S,s(E,v.prototype),E.isPureReactComponent=!0;var L={current:null},M=Object.prototype.hasOwnProperty,k={key:!0,ref:!0,__self:!0,__source:!0};function _(e,t,n){var s,r={},a=null,i=null;if(null!=t)for(s in void 0!==t.ref&&(i=t.ref),void 0!==t.key&&(a=""+t.key),t)M.call(t,s)&&!k.hasOwnProperty(s)&&(r[s]=t[s]);var c=arguments.length-2;if(1===c)r.children=n;else if(1<c){for(var l=Array(c),d=0;d<c;d++)l[d]=arguments[d+2];r.children=l}if(e&&e.defaultProps)for(s in c=e.defaultProps)void 0===r[s]&&(r[s]=c[s]);return{$$typeof:o,type:e,key:a,ref:i,props:r,_owner:L.current}}function w(e){return"object"==typeof e&&null!==e&&e.$$typeof===o}var I=/\/+/g,C=[];function Q(e,t,n,s){if(C.length){var r=C.pop();return r.result=e,r.keyPrefix=t,r.func=n,r.context=s,r.count=0,r}return{result:e,keyPrefix:t,func:n,context:s,count:0}}function R(e){e.result=null,e.keyPrefix=null,e.func=null,e.context=null,e.count=0,10>C.length&&C.push(e)}function D(e,t,n,s){var r=typeof e;"undefined"!==r&&"boolean"!==r||(e=null);var i=!1;if(null===e)i=!0;else switch(r){case"string":case"number":i=!0;break;case"object":switch(e.$$typeof){case o:case a:i=!0}}if(i)return n(s,e,""===t?"."+O(e,0):t),1;if(i=0,t=""===t?".":t+":",Array.isArray(e))for(var c=0;c<e.length;c++){var l=t+O(r=e[c],c);i+=D(r,l,n,s)}else if(null===e||"object"!=typeof e?l=null:l="function"==typeof(l=f&&e[f]||e["@@iterator"])?l:null,"function"==typeof l)for(e=l.call(e),c=0;!(r=e.next()).done;)i+=D(r=r.value,l=t+O(r,c++),n,s);else if("object"===r)throw n=""+e,Error(b(31,"[object Object]"===n?"object with keys {"+Object.keys(e).join(", ")+"}":n,""));return i}function T(e,t,n){return null==e?0:D(e,"",t,n)}function O(e,t){return"object"==typeof e&&null!==e&&null!=e.key?function(e){var t={"=":"=0",":":"=2"};return"$"+(""+e).replace(/[=:]/g,(function(e){return t[e]}))}(e.key):t.toString(36)}function P(e,t){e.func.call(e.context,t,e.count++)}function F(e,t,n){var s=e.result,r=e.keyPrefix;e=e.func.call(e.context,t,e.count++),Array.isArray(e)?N(e,s,n,(function(e){return e})):null!=e&&(w(e)&&(e=function(e,t){return{$$typeof:o,type:e.type,key:t,ref:e.ref,props:e.props,_owner:e._owner}}(e,r+(!e.key||t&&t.key===e.key?"":(""+e.key).replace(I,"$&/")+"/")+n)),s.push(e))}function N(e,t,n,s,r){var o="";null!=n&&(o=(""+n).replace(I,"$&/")+"/"),T(e,F,t=Q(t,o,s,r)),R(t)}var B={current:null};function V(){var e=B.current;if(null===e)throw Error(b(321));return e}var G={ReactCurrentDispatcher:B,ReactCurrentBatchConfig:{suspense:null},ReactCurrentOwner:L,IsSomeRendererActing:{current:!1},assign:s};t.Children={map:function(e,t,n){if(null==e)return e;var s=[];return N(e,s,null,t,n),s},forEach:function(e,t,n){if(null==e)return e;T(e,P,t=Q(null,null,t,n)),R(t)},count:function(e){return T(e,(function(){return null}),null)},toArray:function(e){var t=[];return N(e,t,null,(function(e){return e})),t},only:function(e){if(!w(e))throw Error(b(143));return e}},t.Component=v,t.Fragment=i,t.Profiler=l,t.PureComponent=S,t.StrictMode=c,t.Suspense=m,t.__SECRET_INTERNALS_DO_NOT_USE_OR_YOU_WILL_BE_FIRED=G,t.cloneElement=function(e,t,n){if(null==e)throw Error(b(267,e));var r=s({},e.props),a=e.key,i=e.ref,c=e._owner;if(null!=t){if(void 0!==t.ref&&(i=t.ref,c=L.current),void 0!==t.key&&(a=""+t.key),e.type&&e.type.defaultProps)var l=e.type.defaultProps;for(d in t)M.call(t,d)&&!k.hasOwnProperty(d)&&(r[d]=void 0===t[d]&&void 0!==l?l[d]:t[d])}var d=arguments.length-2;if(1===d)r.children=n;else if(1<d){l=Array(d);for(var p=0;p<d;p++)l[p]=arguments[p+2];r.children=l}return{$$typeof:o,type:e.type,key:a,ref:i,props:r,_owner:c}},t.createContext=function(e,t){return void 0===t&&(t=null),(e={$$typeof:p,_calculateChangedBits:t,_currentValue:e,_currentValue2:e,_threadCount:0,Provider:null,Consumer:null}).Provider={$$typeof:d,_context:e},e.Consumer=e},t.createElement=_,t.createFactory=function(e){var t=_.bind(null,e);return t.type=e,t},t.createRef=function(){return{current:null}},t.forwardRef=function(e){return{$$typeof:u,render:e}},t.isValidElement=w,t.lazy=function(e){return{$$typeof:g,_ctor:e,_status:-1,_result:null}},t.memo=function(e,t){return{$$typeof:h,type:e,compare:void 0===t?null:t}},t.useCallback=function(e,t){return V().useCallback(e,t)},t.useContext=function(e,t){return V().useContext(e,t)},t.useDebugValue=function(){},t.useEffect=function(e,t){return V().useEffect(e,t)},t.useImperativeHandle=function(e,t,n){return V().useImperativeHandle(e,t,n)},t.useLayoutEffect=function(e,t){return V().useLayoutEffect(e,t)},t.useMemo=function(e,t){return V().useMemo(e,t)},t.useReducer=function(e,t,n){return V().useReducer(e,t,n)},t.useRef=function(e){return V().useRef(e)},t.useState=function(e){return V().useState(e)},t.version="16.14.0"},67294:(e,t,n)=>{"use strict";e.exports=n(72408)},60053:(e,t)=>{"use strict";var n,s,r,o,a;if("undefined"==typeof window||"function"!=typeof MessageChannel){var i=null,c=null,l=function(){if(null!==i)try{var e=t.unstable_now();i(!0,e),i=null}catch(n){throw setTimeout(l,0),n}},d=Date.now();t.unstable_now=function(){return Date.now()-d},n=function(e){null!==i?setTimeout(n,0,e):(i=e,setTimeout(l,0))},s=function(e,t){c=setTimeout(e,t)},r=function(){clearTimeout(c)},o=function(){return!1},a=t.unstable_forceFrameRate=function(){}}else{var p=window.performance,u=window.Date,m=window.setTimeout,h=window.clearTimeout;if("undefined"!=typeof console){var g=window.cancelAnimationFrame;"function"!=typeof window.requestAnimationFrame&&console.error("This browser doesn't support requestAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills"),"function"!=typeof g&&console.error("This browser doesn't support cancelAnimationFrame. Make sure that you load a polyfill in older browsers. https://fb.me/react-polyfills")}if("object"==typeof p&&"function"==typeof p.now)t.unstable_now=function(){return p.now()};else{var f=u.now();t.unstable_now=function(){return u.now()-f}}var b=!1,y=null,x=-1,v=5,A=0;o=function(){return t.unstable_now()>=A},a=function(){},t.unstable_forceFrameRate=function(e){0>e||125<e?console.error("forceFrameRate takes a positive int between 0 and 125, forcing framerates higher than 125 fps is not unsupported"):v=0<e?Math.floor(1e3/e):5};var S=new MessageChannel,E=S.port2;S.port1.onmessage=function(){if(null!==y){var e=t.unstable_now();A=e+v;try{y(!0,e)?E.postMessage(null):(b=!1,y=null)}catch(n){throw E.postMessage(null),n}}else b=!1},n=function(e){y=e,b||(b=!0,E.postMessage(null))},s=function(e,n){x=m((function(){e(t.unstable_now())}),n)},r=function(){h(x),x=-1}}function L(e,t){var n=e.length;e.push(t);e:for(;;){var s=n-1>>>1,r=e[s];if(!(void 0!==r&&0<_(r,t)))break e;e[s]=t,e[n]=r,n=s}}function M(e){return void 0===(e=e[0])?null:e}function k(e){var t=e[0];if(void 0!==t){var n=e.pop();if(n!==t){e[0]=n;e:for(var s=0,r=e.length;s<r;){var o=2*(s+1)-1,a=e[o],i=o+1,c=e[i];if(void 0!==a&&0>_(a,n))void 0!==c&&0>_(c,a)?(e[s]=c,e[i]=n,s=i):(e[s]=a,e[o]=n,s=o);else{if(!(void 0!==c&&0>_(c,n)))break e;e[s]=c,e[i]=n,s=i}}}return t}return null}function _(e,t){var n=e.sortIndex-t.sortIndex;return 0!==n?n:e.id-t.id}var w=[],I=[],C=1,Q=null,R=3,D=!1,T=!1,O=!1;function P(e){for(var t=M(I);null!==t;){if(null===t.callback)k(I);else{if(!(t.startTime<=e))break;k(I),t.sortIndex=t.expirationTime,L(w,t)}t=M(I)}}function F(e){if(O=!1,P(e),!T)if(null!==M(w))T=!0,n(N);else{var t=M(I);null!==t&&s(F,t.startTime-e)}}function N(e,n){T=!1,O&&(O=!1,r()),D=!0;var a=R;try{for(P(n),Q=M(w);null!==Q&&(!(Q.expirationTime>n)||e&&!o());){var i=Q.callback;if(null!==i){Q.callback=null,R=Q.priorityLevel;var c=i(Q.expirationTime<=n);n=t.unstable_now(),"function"==typeof c?Q.callback=c:Q===M(w)&&k(w),P(n)}else k(w);Q=M(w)}if(null!==Q)var l=!0;else{var d=M(I);null!==d&&s(F,d.startTime-n),l=!1}return l}finally{Q=null,R=a,D=!1}}function B(e){switch(e){case 1:return-1;case 2:return 250;case 5:return 1073741823;case 4:return 1e4;default:return 5e3}}var V=a;t.unstable_IdlePriority=5,t.unstable_ImmediatePriority=1,t.unstable_LowPriority=4,t.unstable_NormalPriority=3,t.unstable_Profiling=null,t.unstable_UserBlockingPriority=2,t.unstable_cancelCallback=function(e){e.callback=null},t.unstable_continueExecution=function(){T||D||(T=!0,n(N))},t.unstable_getCurrentPriorityLevel=function(){return R},t.unstable_getFirstCallbackNode=function(){return M(w)},t.unstable_next=function(e){switch(R){case 1:case 2:case 3:var t=3;break;default:t=R}var n=R;R=t;try{return e()}finally{R=n}},t.unstable_pauseExecution=function(){},t.unstable_requestPaint=V,t.unstable_runWithPriority=function(e,t){switch(e){case 1:case 2:case 3:case 4:case 5:break;default:e=3}var n=R;R=e;try{return t()}finally{R=n}},t.unstable_scheduleCallback=function(e,o,a){var i=t.unstable_now();if("object"==typeof a&&null!==a){var c=a.delay;c="number"==typeof c&&0<c?i+c:i,a="number"==typeof a.timeout?a.timeout:B(e)}else a=B(e),c=i;return e={id:C++,callback:o,priorityLevel:e,startTime:c,expirationTime:a=c+a,sortIndex:-1},c>i?(e.sortIndex=c,L(I,e),null===M(w)&&e===M(I)&&(O?r():O=!0,s(F,c-i))):(e.sortIndex=a,L(w,e),T||D||(T=!0,n(N))),e},t.unstable_shouldYield=function(){var e=t.unstable_now();P(e);var n=M(w);return n!==Q&&null!==Q&&null!==n&&null!==n.callback&&n.startTime<=e&&n.expirationTime<Q.expirationTime||o()},t.unstable_wrapCallback=function(e){var t=R;return function(){var n=R;R=t;try{return e.apply(this,arguments)}finally{R=n}}}},63840:(e,t,n)=>{"use strict";e.exports=n(60053)},96774:e=>{e.exports=function(e,t,n,s){var r=n?n.call(s,e,t):void 0;if(void 0!==r)return!!r;if(e===t)return!0;if("object"!=typeof e||!e||"object"!=typeof t||!t)return!1;var o=Object.keys(e),a=Object.keys(t);if(o.length!==a.length)return!1;for(var i=Object.prototype.hasOwnProperty.bind(t),c=0;c<o.length;c++){var l=o[c];if(!i(l))return!1;var d=e[l],p=t[l];if(!1===(r=n?n.call(s,d,p,l):void 0)||void 0===r&&d!==p)return!1}return!0}},2177:(e,t,n)=>{"use strict";n.d(t,{Z:()=>r});var s="Invariant failed";function r(e,t){if(!e)throw new Error(s)}},53250:(e,t,n)=>{"use strict";var s=n(67294);var r="function"==typeof Object.is?Object.is:function(e,t){return e===t&&(0!==e||1/e==1/t)||e!=e&&t!=t},o=s.useState,a=s.useEffect,i=s.useLayoutEffect,c=s.useDebugValue;function l(e){var t=e.getSnapshot;e=e.value;try{var n=t();return!r(e,n)}catch(s){return!0}}var d="undefined"==typeof window||void 0===window.document||void 0===window.document.createElement?function(e,t){return t()}:function(e,t){var n=t(),s=o({inst:{value:n,getSnapshot:t}}),r=s[0].inst,d=s[1];return i((function(){r.value=n,r.getSnapshot=t,l(r)&&d({inst:r})}),[e,n,t]),a((function(){return l(r)&&d({inst:r}),e((function(){l(r)&&d({inst:r})}))}),[e]),c(n),n};t.useSyncExternalStore=void 0!==s.useSyncExternalStore?s.useSyncExternalStore:d},61688:(e,t,n)=>{"use strict";e.exports=n(53250)},36809:(e,t,n)=>{"use strict";n.r(t),n.d(t,{default:()=>s});const s={title:"SynapseML",tagline:"Simple and Distributed Machine Learning",url:"https://microsoft.github.io",baseUrl:"/SynapseML/",favicon:"img/favicon.ico",organizationName:"microsoft",projectName:"SynapseML",trailingSlash:!0,customFields:{version:"1.0.4"},stylesheets:[{href:"https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css",integrity:"sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc",crossorigin:"anonymous"}],themeConfig:{prism:{theme:{plain:{color:"#393A34",backgroundColor:"#f6f8fa"},styles:[{types:["comment","prolog","doctype","cdata"],style:{color:"#999988",fontStyle:"italic"}},{types:["namespace"],style:{opacity:.7}},{types:["string","attr-value"],style:{color:"#e3116c"}},{types:["punctuation","operator"],style:{color:"#393A34"}},{types:["entity","url","symbol","number","boolean","variable","constant","property","regex","inserted"],style:{color:"#36acaa"}},{types:["atrule","keyword","attr-name","selector"],style:{color:"#00a4db"}},{types:["function","deleted","tag"],style:{color:"#d73a49"}},{types:["function-variable"],style:{color:"#6f42c1"}},{types:["tag","selector","keyword"],style:{color:"#00009f"}}]},darkTheme:{plain:{color:"#f8f8f2",backgroundColor:"#272822"},styles:[{types:["comment"],style:{color:"rgb(136, 132, 111)"}},{types:["string","changed"],style:{color:"rgb(230, 219, 116)"}},{types:["punctuation","tag","deleted"],style:{color:"rgb(249, 38, 114)"}},{types:["number","builtin"],style:{color:"rgb(174, 129, 255)"}},{types:["variable"],style:{color:"rgb(248, 248, 242)"}},{types:["function","attr-name","inserted"],style:{color:"rgb(166, 226, 46)"}}]},additionalLanguages:["csharp","powershell"],magicComments:[{className:"theme-code-block-highlighted-line",line:"highlight-next-line",block:{start:"highlight-start",end:"highlight-end"}}]},colorMode:{defaultMode:"dark",disableSwitch:!1,respectPrefersColorScheme:!1},image:"img/synapseml_og.jpg",navbar:{title:"SynapseML",logo:{alt:"SynapseML Logo",src:"img/logo.svg"},items:[{to:"docs/Overview",label:"Docs",position:"left"},{to:"blog",label:"Blog",position:"left"},{to:"videos",label:"Videos",position:"left"},{type:"docsVersionDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{type:"localeDropdown",position:"right",dropdownItemsBefore:[],dropdownItemsAfter:[]},{label:"Developer Docs",position:"right",items:[{label:"Python",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html"},{label:"Scala",href:"https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html"}]},{href:"https://github.com/microsoft/SynapseML",position:"right",className:"header-github-link","aria-label":"GitHub repository"}],hideOnScroll:!1},footer:{style:"dark",links:[{title:"Docs",items:[{label:"Installation",to:"docs/Get%20Started/Install%20SynapseML"},{label:"Getting Started",to:"docs/Get%20Started/Quickstart%20-%20Your%20First%20Models"},{label:"Python API Reference",to:"https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html"},{label:"Scala API Reference",to:"https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html"}]},{title:"More",items:[{label:"Blog",to:"blog"},{label:"Videos",to:"videos"}]},{title:"Community",items:[{label:"GitHub",href:"https://github.com/microsoft/SynapseML"}]}],copyright:"Copyright \xa9 2024 Microsoft."},algolia:{appId:"GBW8AA15RD",apiKey:"70a6807005c645678741ab941bb89ed8",indexName:"synapseML",contextualSearch:!0,searchParameters:{},searchPagePath:"search"},announcementBar:{id:"announcementBar-1",content:'\u2b50\ufe0f If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> \u2b50',isCloseable:!0},docs:{versionPersistence:"localStorage",sidebar:{hideable:!1,autoCollapseCategories:!1}},metadata:[],tableOfContents:{minHeadingLevel:2,maxHeadingLevel:3}},presets:[["@docusaurus/preset-classic",{docs:{sidebarPath:"/home/vsts/work/1/s/website/sidebars.js",remarkPlugins:[null],rehypePlugins:[null]},theme:{customCss:"/home/vsts/work/1/s/website/src/css/custom.css"},gtag:{trackingID:"G-RWPE0183E8",anonymizeIP:!0},blog:{feedOptions:{type:"all"}}}]],plugins:[["@docusaurus/plugin-client-redirects",{redirects:[{to:"/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Create%20Audiobooks/"},{to:"/docs/Overview/",from:"/docs/about/"},{to:"/docs/Explore Algorithms/OpenAI/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20OpenAI/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/features/lightgbm/about/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/getting_started/installation/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",from:"/docs/features/isolation_forest/IsolationForest%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Quick Examples/transformers/transformers_cognitive/",from:"/docs/documentation/transformers/transformers_cognitive/"},{to:"/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",from:"/docs/features/onnx/ONNX%20-%20Inference%20on%20Spark/"},{to:"/docs/Explore Algorithms/AI Services/Geospatial Services/",from:"/docs/features/geospatial_services/GeospatialServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/",from:"/docs/features/hyperparameter_tuning/HyperOpt-SynapseML/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Overview/",from:"/docs/features/vw/Vowpal%20Wabbit%20-%20Overview/"},{to:"/docs/Reference/Developer Setup/",from:"/docs/reference/developer-readme/"},{to:"/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/",from:"/docs/features/responsible_ai/Data%20Balance%20Analysis/"},{to:"/docs/Use with MLFlow/Overview/",from:"/docs/next/mlflow/examples/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/0.10.1/getting_started/installation/"},{to:"/docs/Explore Algorithms/Deep Learning/ONNX/",from:"/docs/next/features/onnx/about/"},{to:"/docs/Explore Algorithms/Causal Inference/Overview/",from:"/docs/features/causal_inference/about/"},{to:"/docs/Quick Examples/transformers/transformers_cognitive/",from:"/docs/next/documentation/transformers/transformers_cognitive/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",from:"/docs/next/features/isolation_forest/IsolationForest%20-%20Multivariate%20Anomaly%20Detection/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/next/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/getting_started/first_model/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/",from:"/docs/next/features/onnx/ONNX%20-%20Inference%20on%20Spark/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/next/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",from:"/docs/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/"},{to:"/docs/Overview/",from:"/docs/next/about/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.11.0/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/getting_started/first_example/"},{to:"/docs/Use with MLFlow/Overview/",from:"/docs/mlflow/examples/"},{to:"/docs/Reference/Dotnet Setup/",from:"/docs/reference/dotnet-setup/"},{to:"/docs/Reference/Quickstart - LightGBM in Dotnet/",from:"/docs/0.10.0/getting_started/dotnet_example/"},{to:"/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/",from:"/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/"},{to:"/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/",from:"/docs/features/other/HyperParameterTuning%20-%20Fighting%20Breast%20Cancer/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/next/features/lightgbm/about/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/next/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Reference/Dotnet Setup/",from:"/docs/next/reference/dotnet-setup/"},{to:"/docs/Overview/",from:"/docs/0.10.0/about/"},{to:"/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/",from:"/docs/0.10.0/features/responsible_ai/Interpretability%20-%20PDP%20and%20ICE%20explainer/"},{to:"/docs/Explore Algorithms/OpenCV/Image Transformations/",from:"/docs/features/opencv/OpenCV%20-%20Pipeline%20Image%20Transformations/"},{to:"/docs/Overview/",from:"/docs/features/spark_serving/about/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.10.1/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.11.1/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.9.4/features/lightgbm/LightGBM%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Tabular Explainers/",from:"/docs/0.9.4/features/responsible_ai/Interpretability%20-%20Tabular%20SHAP%20explainer/"},{to:"/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/",from:"/docs/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/"},{to:"/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/",from:"/docs/features/classification/Classification%20-%20Adult%20Census/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/",from:"/docs/features/vw/Vowpal%20Wabbit%20-%20Contextual%20Bandits/"},{to:"/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/",from:"/docs/next/features/simple_deep_learning/DeepLearning%20-%20Deep%20Text%20Classification/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/next/getting_started/installation/"},{to:"/docs/Explore Algorithms/LightGBM/Overview/",from:"/docs/0.10.1/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Use with MLFlow/Autologging/",from:"/docs/0.10.1/mlflow/autologging/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/0.9.4/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/0.9.5/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Quick Examples/estimators/estimators_cognitive/",from:"/docs/documentation/estimators/estimators_cognitive/"},{to:"/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/",from:"/docs/features/cognitive_services/CognitiveServices%20-%20Create%20a%20Multilingual%20Search%20Engine%20from%20Forms/"},{to:"/docs/Quick Examples/estimators/estimators_lightgbm/",from:"/docs/next/documentation/estimators/estimators_lightgbm/"},{to:"/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/",from:"/docs/0.9.4/features/responsible_ai/Interpretability%20-%20Explanation%20Dashboard/"},{to:"/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/",from:"/docs/0.9.5/features/responsible_ai/Model%20Interpretation%20on%20Spark/"},{to:"/docs/Explore Algorithms/OpenAI/",from:"/docs/Explore%20Algorithms/OpenAI/"},{to:"/docs/Explore Algorithms/OpenCV/Image Transformations/",from:"/docs/documentation/transformers/transformers_opencv/"},{to:"/docs/Explore Algorithms/AI Services/Overview/",from:"/docs/0.10.0/features/cognitive_services/CognitiveServices%20-%20Overview/"},{to:"/docs/Use with MLFlow/Autologging/",from:"/docs/0.10.0/mlflow/autologging/"},{to:"/docs/Overview/",from:"/docs/0.11.0/about/"},{to:"/docs/Explore Algorithms/Vowpal Wabbit/Overview/",from:"/docs/0.11.1/features/vw/Vowpal%20Wabbit%20-%20Overview/"},{to:"/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",from:"/docs/0.9.4/features/responsible_ai/Data%20Balance%20Analysis/"},{to:"/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/",from:"/docs/0.9.4/features/responsible_ai/DataBalanceAnalysis%20-%20Adult%20Census%20Income/"},{to:"/docs/Get Started/Install SynapseML/",from:"/docs/Get%20Started/Install%20SynapseML/"},{to:"/docs/Quick Examples/transformers/transformers_core/",from:"/docs/next/documentation/transformers/transformers_core/"},{to:"/docs/Get Started/Quickstart - Your First Models/",from:"/docs/Get%20Started/Quickstart%20-%20Your%20First%20Models/"},{to:"/docs/Explore Algorithms/Deep Learning/Getting Started/",from:"/docs/features/simple_deep_learning/about/"},{to:"/docs/Explore Algorithms/AI Services/Geospatial Services/",from:"/docs/next/features/geospatial_services/GeospatialServices%20-%20Overview/"}]}]],baseUrlIssueBanner:!0,i18n:{defaultLocale:"en",path:"i18n",locales:["en"],localeConfigs:{}},onBrokenLinks:"throw",onBrokenMarkdownLinks:"warn",onDuplicateRoutes:"warn",staticDirectories:["static"],themes:[],scripts:[],headTags:[],clientModules:[],titleDelimiter:"|",noIndex:!1,markdown:{mermaid:!1}}},83117:(e,t,n)=>{"use strict";function s(){return s=Object.assign?Object.assign.bind():function(e){for(var t=1;t<arguments.length;t++){var n=arguments[t];for(var s in n)Object.prototype.hasOwnProperty.call(n,s)&&(e[s]=n[s])}return e},s.apply(this,arguments)}n.d(t,{Z:()=>s})},90144:(e,t,n)=>{"use strict";function s(e,t){return s=Object.setPrototypeOf?Object.setPrototypeOf.bind():function(e,t){return e.__proto__=t,e},s(e,t)}function r(e,t){e.prototype=Object.create(t.prototype),e.prototype.constructor=e,s(e,t)}n.d(t,{Z:()=>r})},80102:(e,t,n)=>{"use strict";function s(e,t){if(null==e)return{};var n,s,r={},o=Object.keys(e);for(s=0;s<o.length;s++)n=o[s],t.indexOf(n)>=0||(r[n]=e[n]);return r}n.d(t,{Z:()=>s})},57529:e=>{"use strict";e.exports={}},16887:e=>{"use strict";e.exports=JSON.parse('{"/SynapseML/blog/-32c":{"__comp":"a6aa9e1f","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","items":[{"content":"c3c516ff"},{"content":"b72abe57"},{"content":"8cb11643"},{"content":"2b30bc14"},{"content":"ab9a8589"},{"content":"2d5b0e6a"}],"metadata":"6a95f87e"},"/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/-ecd":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"e1b2b5f3"},"/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/-870":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"e2ce57ed"},"/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/-83d":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"1abf1e18"},"/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/-347":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"b962e007"},"/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/-546":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"561bd03d"},"/SynapseML/blog/archive/-140":{"__comp":"9e4087bc","__context":{"plugin":"3fb29942"},"archive":"d175df5e"},"/SynapseML/blog/overview/-32f":{"__comp":"ccc49370","__context":{"plugin":"3fb29942"},"sidebar":"814f3328","content":"cb5f1a06"},"/SynapseML/search/-a70":{"__comp":"1a4e3797","__context":{"plugin":"fc338b02"}},"/SynapseML/videos/-9fc":{"__comp":"34f00221","__context":{"plugin":"627aefca"},"config":"5e9f5e1a"},"/SynapseML/docs/0.11.3/-649":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"ffb7f88a"},"/SynapseML/docs/0.11.3/Deploy Models/Overview/-107":{"__comp":"17896441","content":"15a59c33"},"/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/-7ca":{"__comp":"17896441","content":"37ab9beb"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-428":{"__comp":"17896441","content":"1e57c92a"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/-20d":{"__comp":"17896441","content":"6b9bdd6f"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-dd0":{"__comp":"17896441","content":"4bbbdfcf"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/-8b1":{"__comp":"17896441","content":"12d85ab5"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-7f9":{"__comp":"17896441","content":"47bbce76"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/-9df":{"__comp":"17896441","content":"cb7c2a83"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/-398":{"__comp":"17896441","content":"707d2a35"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-475":{"__comp":"17896441","content":"b80b493a"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-746":{"__comp":"17896441","content":"13f17286"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-00a":{"__comp":"17896441","content":"d25cde70"},"/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-be4":{"__comp":"17896441","content":"900b935c"},"/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-4ed":{"__comp":"17896441","content":"ae89cd3b"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/-f94":{"__comp":"17896441","content":"efbc69e1"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-af8":{"__comp":"17896441","content":"b8963dc0"},"/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-c3d":{"__comp":"17896441","content":"93ba15e0"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-1f2":{"__comp":"17896441","content":"20641120"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/-d3c":{"__comp":"17896441","content":"fd4bd09d"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-b90":{"__comp":"17896441","content":"86ed1ff2"},"/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-583":{"__comp":"17896441","content":"327be84b"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/-c90":{"__comp":"17896441","content":"04ee4ab4"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/-59d":{"__comp":"17896441","content":"3a4f9d93"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/-4d0":{"__comp":"17896441","content":"0f113696"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-b98":{"__comp":"17896441","content":"ba9c0924"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-52d":{"__comp":"17896441","content":"ee83122d"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-018":{"__comp":"17896441","content":"6ff7775f"},"/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-086":{"__comp":"17896441","content":"deef0af8"},"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-2e6":{"__comp":"17896441","content":"ed316aed"},"/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-cd5":{"__comp":"17896441","content":"6b40ec54"},"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/-7d2":{"__comp":"17896441","content":"391cb159"},"/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-c77":{"__comp":"17896441","content":"e579bad6"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/-f47":{"__comp":"17896441","content":"e9b75d67"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/-ffd":{"__comp":"17896441","content":"e3089280"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-c37":{"__comp":"17896441","content":"5f17911b"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-3dc":{"__comp":"17896441","content":"6f8cd013"},"/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/-5b7":{"__comp":"17896441","content":"da508cef"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/-e89":{"__comp":"17896441","content":"4e3910af"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-22c":{"__comp":"17896441","content":"b328a361"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-6da":{"__comp":"17896441","content":"4de42975"},"/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-4f5":{"__comp":"17896441","content":"995576e9"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/-b53":{"__comp":"17896441","content":"1dc2f362"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/-e82":{"__comp":"17896441","content":"53a5cb1e"},"/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-eb4":{"__comp":"17896441","content":"109f6864"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/-aa0":{"__comp":"17896441","content":"59ef8022"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/-547":{"__comp":"17896441","content":"052105dd"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/-ca9":{"__comp":"17896441","content":"6f479459"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-1c8":{"__comp":"17896441","content":"4997ef4f"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-850":{"__comp":"17896441","content":"5066efb2"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-103":{"__comp":"17896441","content":"c065fcb9"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-22a":{"__comp":"17896441","content":"255635fc"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/-4a3":{"__comp":"17896441","content":"79114dfe"},"/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/-51c":{"__comp":"17896441","content":"50af03e4"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-79d":{"__comp":"17896441","content":"9b020c4a"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-dbd":{"__comp":"17896441","content":"fba72e87"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/-f6c":{"__comp":"17896441","content":"13246fd5"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-899":{"__comp":"17896441","content":"0fa36e32"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-de1":{"__comp":"17896441","content":"295a8e78"},"/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-6e5":{"__comp":"17896441","content":"0606f1a5"},"/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/-604":{"__comp":"17896441","content":"82878c84"},"/SynapseML/docs/0.11.3/Get Started/Install SynapseML/-53d":{"__comp":"17896441","content":"d3c9b0d7"},"/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/-bcf":{"__comp":"17896441","content":"c4d09a44"},"/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/-e8a":{"__comp":"17896441","content":"0a65b9de"},"/SynapseML/docs/0.11.3/Overview/-201":{"__comp":"17896441","content":"09bc1516"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/-f33":{"__comp":"17896441","content":"c40984d1"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/-d03":{"__comp":"17896441","content":"325e22bd"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/-0fc":{"__comp":"17896441","content":"5962ef6e"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/-113":{"__comp":"17896441","content":"4e044b4e"},"/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/-bcf":{"__comp":"17896441","content":"a877f9e5"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/-3f4":{"__comp":"17896441","content":"0d6d64cd"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/-abc":{"__comp":"17896441","content":"9d095dba"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/-b7d":{"__comp":"17896441","content":"48329b94"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/-a66":{"__comp":"17896441","content":"ad10988d"},"/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/-b96":{"__comp":"17896441","content":"548964ce"},"/SynapseML/docs/0.11.3/Reference/Contributor Guide/-06f":{"__comp":"17896441","content":"b41f295c"},"/SynapseML/docs/0.11.3/Reference/Developer Setup/-cfc":{"__comp":"17896441","content":"8e32e44d"},"/SynapseML/docs/0.11.3/Reference/Docker Setup/-419":{"__comp":"17896441","content":"57e687e8"},"/SynapseML/docs/0.11.3/Reference/Dotnet Setup/-46d":{"__comp":"17896441","content":"603706f6"},"/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/-9b3":{"__comp":"17896441","content":"71042800"},"/SynapseML/docs/0.11.3/Reference/R Setup/-1a7":{"__comp":"17896441","content":"04fc65cc"},"/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/-6a1":{"__comp":"17896441","content":"9c5088a2"},"/SynapseML/docs/0.11.3/Use with MLFlow/Install/-aa4":{"__comp":"17896441","content":"d39aa6d3"},"/SynapseML/docs/0.11.3/Use with MLFlow/Overview/-a43":{"__comp":"17896441","content":"1587ba1d"},"/SynapseML/docs/0.11.4/-72b":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"4e432063"},"/SynapseML/docs/0.11.4/Deploy Models/Overview/-1a4":{"__comp":"17896441","content":"a9330763"},"/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/-ceb":{"__comp":"17896441","content":"ad139e3c"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-f98":{"__comp":"17896441","content":"cf57716c"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/-9e7":{"__comp":"17896441","content":"e21a3367"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-2c3":{"__comp":"17896441","content":"6e159789"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/-be8":{"__comp":"17896441","content":"77c259dd"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-1d4":{"__comp":"17896441","content":"fd3b6ea5"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/-31e":{"__comp":"17896441","content":"01a15f20"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/-8c4":{"__comp":"17896441","content":"74659d33"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-4f9":{"__comp":"17896441","content":"0637d1e7"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-d9d":{"__comp":"17896441","content":"f322d2da"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5a7":{"__comp":"17896441","content":"a406e231"},"/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-e8f":{"__comp":"17896441","content":"e8b9c2c7"},"/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-8e2":{"__comp":"17896441","content":"f8597cb0"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/-78b":{"__comp":"17896441","content":"8fd0c721"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-756":{"__comp":"17896441","content":"22f921cc"},"/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-264":{"__comp":"17896441","content":"e4e95844"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-c8d":{"__comp":"17896441","content":"be95fd8f"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/-247":{"__comp":"17896441","content":"fe2c893b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-c00":{"__comp":"17896441","content":"b7802ae1"},"/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-c42":{"__comp":"17896441","content":"aba932dd"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/-53f":{"__comp":"17896441","content":"ef6ec597"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/-2d6":{"__comp":"17896441","content":"8181e18a"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/-169":{"__comp":"17896441","content":"70b306ba"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-b68":{"__comp":"17896441","content":"67e06a8b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9c7":{"__comp":"17896441","content":"061b38c5"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-ff8":{"__comp":"17896441","content":"0daa5b3f"},"/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-016":{"__comp":"17896441","content":"a34c49d0"},"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-6a6":{"__comp":"17896441","content":"e4ba4487"},"/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-b11":{"__comp":"17896441","content":"4cee39d7"},"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/-e22":{"__comp":"17896441","content":"d9837698"},"/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-934":{"__comp":"17896441","content":"a608660b"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/-fea":{"__comp":"17896441","content":"c5775233"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/-cfe":{"__comp":"17896441","content":"a0434473"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-78f":{"__comp":"17896441","content":"0453ae68"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-163":{"__comp":"17896441","content":"c413b43a"},"/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/-29e":{"__comp":"17896441","content":"c73cfc52"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/-747":{"__comp":"17896441","content":"9fa7e8eb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-faf":{"__comp":"17896441","content":"bf002efe"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-566":{"__comp":"17896441","content":"0e16e9ea"},"/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-491":{"__comp":"17896441","content":"d620a661"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/-7a4":{"__comp":"17896441","content":"4d31bfb3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/-75f":{"__comp":"17896441","content":"c991f47b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-2b0":{"__comp":"17896441","content":"15ea8379"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/-f95":{"__comp":"17896441","content":"60a2189a"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/-66b":{"__comp":"17896441","content":"7a1d9beb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/-e63":{"__comp":"17896441","content":"49e6864b"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-783":{"__comp":"17896441","content":"31ee0fa3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-e01":{"__comp":"17896441","content":"db170a23"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-3b8":{"__comp":"17896441","content":"38d6824e"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-80b":{"__comp":"17896441","content":"069e3c4c"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/-18e":{"__comp":"17896441","content":"8328d740"},"/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/-128":{"__comp":"17896441","content":"11c317cb"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-02f":{"__comp":"17896441","content":"dea122e2"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-6d4":{"__comp":"17896441","content":"0e2bf69e"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/-b97":{"__comp":"17896441","content":"8af72580"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-6ca":{"__comp":"17896441","content":"ceeda143"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-c86":{"__comp":"17896441","content":"d41234a3"},"/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-964":{"__comp":"17896441","content":"b940e0b5"},"/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/-cb4":{"__comp":"17896441","content":"c2492d56"},"/SynapseML/docs/0.11.4/Get Started/Install SynapseML/-34f":{"__comp":"17896441","content":"034e6169"},"/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/-6fa":{"__comp":"17896441","content":"0e0ee9b2"},"/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/-528":{"__comp":"17896441","content":"767a7177"},"/SynapseML/docs/0.11.4/Overview/-83a":{"__comp":"17896441","content":"b9f125cc"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/-32e":{"__comp":"17896441","content":"589adaf2"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/-269":{"__comp":"17896441","content":"965ed185"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/-11e":{"__comp":"17896441","content":"c90c942b"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/-c94":{"__comp":"17896441","content":"d1246f49"},"/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/-bf0":{"__comp":"17896441","content":"1842239f"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/-728":{"__comp":"17896441","content":"dac82c00"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/-12e":{"__comp":"17896441","content":"e24ee294"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/-1cc":{"__comp":"17896441","content":"3ac37a06"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/-8fb":{"__comp":"17896441","content":"f7bfff73"},"/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/-828":{"__comp":"17896441","content":"df01388f"},"/SynapseML/docs/0.11.4/Reference/Contributor Guide/-881":{"__comp":"17896441","content":"e9aab890"},"/SynapseML/docs/0.11.4/Reference/Developer Setup/-171":{"__comp":"17896441","content":"dd29718b"},"/SynapseML/docs/0.11.4/Reference/Docker Setup/-f85":{"__comp":"17896441","content":"e5c4ef01"},"/SynapseML/docs/0.11.4/Reference/Dotnet Setup/-f28":{"__comp":"17896441","content":"7aa7b8e0"},"/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/-553":{"__comp":"17896441","content":"0b416bde"},"/SynapseML/docs/0.11.4/Reference/R Setup/-fd3":{"__comp":"17896441","content":"85aed2f9"},"/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/-3cb":{"__comp":"17896441","content":"20e0fe38"},"/SynapseML/docs/0.11.4/Use with MLFlow/Install/-360":{"__comp":"17896441","content":"3c1f4383"},"/SynapseML/docs/0.11.4/Use with MLFlow/Overview/-4e1":{"__comp":"17896441","content":"d355047f"},"/SynapseML/docs/1.0.1/-c9e":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"cd8e97a5"},"/SynapseML/docs/1.0.1/Deploy Models/Overview/-7aa":{"__comp":"17896441","content":"aa5548ab"},"/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/-c4a":{"__comp":"17896441","content":"179d57fa"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-d96":{"__comp":"17896441","content":"e1269457"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/-a96":{"__comp":"17896441","content":"ee6cea3b"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-4e5":{"__comp":"17896441","content":"4e105cba"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/-a84":{"__comp":"17896441","content":"70fe7dd7"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-e94":{"__comp":"17896441","content":"990f07de"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/-be5":{"__comp":"17896441","content":"7bf16cc6"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-76f":{"__comp":"17896441","content":"1155dc6f"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-5fa":{"__comp":"17896441","content":"4bea8531"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-417":{"__comp":"17896441","content":"6c7d288d"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-cb0":{"__comp":"17896441","content":"296054a7"},"/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-831":{"__comp":"17896441","content":"ca0a1bad"},"/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-ce6":{"__comp":"17896441","content":"166d99f5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/-a37":{"__comp":"17896441","content":"336404dc"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-0db":{"__comp":"17896441","content":"e887b273"},"/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-d46":{"__comp":"17896441","content":"e7250a93"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-b74":{"__comp":"17896441","content":"00ab2cda"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/-109":{"__comp":"17896441","content":"2ece0fbe"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-de7":{"__comp":"17896441","content":"bece3771"},"/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-250":{"__comp":"17896441","content":"ac581902"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/-68a":{"__comp":"17896441","content":"d252041b"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/-920":{"__comp":"17896441","content":"99e5ffa1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/-4a5":{"__comp":"17896441","content":"3b5de274"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-a53":{"__comp":"17896441","content":"e316de39"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9d3":{"__comp":"17896441","content":"39122aab"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-da8":{"__comp":"17896441","content":"5cfa133c"},"/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-b25":{"__comp":"17896441","content":"a07d09c1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-18d":{"__comp":"17896441","content":"4db82253"},"/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-f90":{"__comp":"17896441","content":"cff2e56e"},"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/-15c":{"__comp":"17896441","content":"993c5022"},"/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-796":{"__comp":"17896441","content":"c2e40c34"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/-045":{"__comp":"17896441","content":"32fe2e34"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/-5e3":{"__comp":"17896441","content":"62983deb"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-6f5":{"__comp":"17896441","content":"6a26e359"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-457":{"__comp":"17896441","content":"7649ec47"},"/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/-f73":{"__comp":"17896441","content":"e1ed1e14"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/-0f7":{"__comp":"17896441","content":"bef0d2d3"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-fa5":{"__comp":"17896441","content":"ed5c6c48"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-201":{"__comp":"17896441","content":"d3093636"},"/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-bd7":{"__comp":"17896441","content":"3201258e"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/-979":{"__comp":"17896441","content":"99f8fee5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/-810":{"__comp":"17896441","content":"3cbfbf75"},"/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-1e9":{"__comp":"17896441","content":"b0e28723"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/-6f0":{"__comp":"17896441","content":"90742975"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/-55c":{"__comp":"17896441","content":"00056db7"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/-e7f":{"__comp":"17896441","content":"b33598ac"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-df0":{"__comp":"17896441","content":"6eddee4c"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-94a":{"__comp":"17896441","content":"54287b47"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-41d":{"__comp":"17896441","content":"6c4c8509"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-d72":{"__comp":"17896441","content":"4130db01"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/-895":{"__comp":"17896441","content":"3a1147b5"},"/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/-b41":{"__comp":"17896441","content":"866b91a1"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-c46":{"__comp":"17896441","content":"22444eb9"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-728":{"__comp":"17896441","content":"1a3608a9"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/-21f":{"__comp":"17896441","content":"12b89d4a"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-a43":{"__comp":"17896441","content":"f8aab3ca"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-7db":{"__comp":"17896441","content":"78aebd5d"},"/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-9d2":{"__comp":"17896441","content":"c14fdd92"},"/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/-697":{"__comp":"17896441","content":"423540f2"},"/SynapseML/docs/1.0.1/Get Started/Install SynapseML/-349":{"__comp":"17896441","content":"fc47fc53"},"/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/-d39":{"__comp":"17896441","content":"90f00051"},"/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/-2fa":{"__comp":"17896441","content":"0d432c84"},"/SynapseML/docs/1.0.1/Overview/-64e":{"__comp":"17896441","content":"1ad91055"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/-8d4":{"__comp":"17896441","content":"6a9ad78d"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/-d28":{"__comp":"17896441","content":"d9e5e528"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/-1b9":{"__comp":"17896441","content":"494501bd"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/-b23":{"__comp":"17896441","content":"445a1b28"},"/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/-86c":{"__comp":"17896441","content":"e7bd83ca"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/-03e":{"__comp":"17896441","content":"001aea4c"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/-0db":{"__comp":"17896441","content":"fece3c24"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/-f4e":{"__comp":"17896441","content":"b38eec89"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/-945":{"__comp":"17896441","content":"a3fd5a6d"},"/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/-369":{"__comp":"17896441","content":"ebf648dd"},"/SynapseML/docs/1.0.1/Reference/Contributor Guide/-5ab":{"__comp":"17896441","content":"76b137bf"},"/SynapseML/docs/1.0.1/Reference/Developer Setup/-971":{"__comp":"17896441","content":"b7c6295c"},"/SynapseML/docs/1.0.1/Reference/Docker Setup/-e6e":{"__comp":"17896441","content":"345903d4"},"/SynapseML/docs/1.0.1/Reference/Dotnet Setup/-179":{"__comp":"17896441","content":"5887dc62"},"/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/-049":{"__comp":"17896441","content":"077ca2ec"},"/SynapseML/docs/1.0.1/Reference/R Setup/-392":{"__comp":"17896441","content":"b7787e72"},"/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/-23e":{"__comp":"17896441","content":"f7580424"},"/SynapseML/docs/1.0.1/Use with MLFlow/Install/-0df":{"__comp":"17896441","content":"35826f14"},"/SynapseML/docs/1.0.1/Use with MLFlow/Overview/-d32":{"__comp":"17896441","content":"4f1ca6a9"},"/SynapseML/docs/1.0.2/-df4":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"dc1e40d7"},"/SynapseML/docs/1.0.2/Deploy Models/Overview/-a7c":{"__comp":"17896441","content":"e12b4691"},"/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/-7f9":{"__comp":"17896441","content":"efc1978d"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-185":{"__comp":"17896441","content":"b1706689"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/-fa7":{"__comp":"17896441","content":"5c18deb5"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-1b3":{"__comp":"17896441","content":"5ef28062"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/-78e":{"__comp":"17896441","content":"d989c707"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-400":{"__comp":"17896441","content":"c23b5e26"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/-5da":{"__comp":"17896441","content":"fd748117"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-286":{"__comp":"17896441","content":"a34b09d4"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-c7f":{"__comp":"17896441","content":"514f485e"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-1d5":{"__comp":"17896441","content":"7c007ec6"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-d5d":{"__comp":"17896441","content":"b066233f"},"/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-46e":{"__comp":"17896441","content":"f1b2dc7f"},"/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-4ab":{"__comp":"17896441","content":"c34b1e36"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/-8e5":{"__comp":"17896441","content":"9cb6fa7c"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-405":{"__comp":"17896441","content":"093b81f4"},"/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-2d7":{"__comp":"17896441","content":"00353f68"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-764":{"__comp":"17896441","content":"95d9e891"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/-f3b":{"__comp":"17896441","content":"f21c13c4"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-8c6":{"__comp":"17896441","content":"513002de"},"/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-40b":{"__comp":"17896441","content":"e8d66fc1"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/-a43":{"__comp":"17896441","content":"b66e2100"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/-001":{"__comp":"17896441","content":"232d09b0"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/-a2e":{"__comp":"17896441","content":"6e71bda3"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-613":{"__comp":"17896441","content":"1e0d2d3b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-5c5":{"__comp":"17896441","content":"76d23901"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-893":{"__comp":"17896441","content":"bc4bf151"},"/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-bb7":{"__comp":"17896441","content":"18c7d2a3"},"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-ccf":{"__comp":"17896441","content":"35bdc661"},"/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-a9c":{"__comp":"17896441","content":"6c7ccb6d"},"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/-108":{"__comp":"17896441","content":"6bdbf6b8"},"/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-a01":{"__comp":"17896441","content":"a8456d9f"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/-f2d":{"__comp":"17896441","content":"3e1e84da"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/-a4a":{"__comp":"17896441","content":"7c2e1100"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-a0a":{"__comp":"17896441","content":"16d05917"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-5bb":{"__comp":"17896441","content":"d0cb74df"},"/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/-6ae":{"__comp":"17896441","content":"a2b5e5df"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/-4c7":{"__comp":"17896441","content":"1014102e"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-056":{"__comp":"17896441","content":"c38c658c"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-a63":{"__comp":"17896441","content":"c3b1d949"},"/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-1e8":{"__comp":"17896441","content":"b75118f0"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/-6cd":{"__comp":"17896441","content":"1a215693"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/-318":{"__comp":"17896441","content":"af1e70b6"},"/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-a37":{"__comp":"17896441","content":"75406112"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/-eed":{"__comp":"17896441","content":"80472ecc"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/-25a":{"__comp":"17896441","content":"18c9c2ba"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/-810":{"__comp":"17896441","content":"567cd2fd"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-5c9":{"__comp":"17896441","content":"e2689dfc"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-b99":{"__comp":"17896441","content":"3647ac9b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-3dd":{"__comp":"17896441","content":"4ef2f7cf"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-1bd":{"__comp":"17896441","content":"88b63415"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/-0b3":{"__comp":"17896441","content":"b576c645"},"/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/-250":{"__comp":"17896441","content":"c2e57a1b"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-bd7":{"__comp":"17896441","content":"d535b358"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-b2f":{"__comp":"17896441","content":"403e642d"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/-5a7":{"__comp":"17896441","content":"445012ce"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-4ab":{"__comp":"17896441","content":"ff428354"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-d16":{"__comp":"17896441","content":"03d0d544"},"/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-157":{"__comp":"17896441","content":"73cf52e8"},"/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/-3f8":{"__comp":"17896441","content":"a0d8a1a2"},"/SynapseML/docs/1.0.2/Get Started/Install SynapseML/-01e":{"__comp":"17896441","content":"98258c5e"},"/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/-0aa":{"__comp":"17896441","content":"268b7b40"},"/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/-095":{"__comp":"17896441","content":"92ce4fd5"},"/SynapseML/docs/1.0.2/Overview/-c78":{"__comp":"17896441","content":"b4898d44"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/-348":{"__comp":"17896441","content":"6c61f93b"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/-386":{"__comp":"17896441","content":"56d039a1"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/-585":{"__comp":"17896441","content":"8462374b"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/-fa7":{"__comp":"17896441","content":"f14633e7"},"/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/-b73":{"__comp":"17896441","content":"3e9a7422"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/-762":{"__comp":"17896441","content":"10e13725"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/-7ec":{"__comp":"17896441","content":"f38a0cbd"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/-bad":{"__comp":"17896441","content":"d0c5e4fa"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/-183":{"__comp":"17896441","content":"39582d99"},"/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/-029":{"__comp":"17896441","content":"07f6b90a"},"/SynapseML/docs/1.0.2/Reference/Contributor Guide/-8f1":{"__comp":"17896441","content":"7b60d8e3"},"/SynapseML/docs/1.0.2/Reference/Developer Setup/-5bf":{"__comp":"17896441","content":"9e5b788f"},"/SynapseML/docs/1.0.2/Reference/Docker Setup/-294":{"__comp":"17896441","content":"1789daeb"},"/SynapseML/docs/1.0.2/Reference/Dotnet Setup/-b72":{"__comp":"17896441","content":"012c1c0f"},"/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/-595":{"__comp":"17896441","content":"ef77ef39"},"/SynapseML/docs/1.0.2/Reference/R Setup/-a22":{"__comp":"17896441","content":"74091a62"},"/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/-e9d":{"__comp":"17896441","content":"25fb872c"},"/SynapseML/docs/1.0.2/Use with MLFlow/Install/-7d6":{"__comp":"17896441","content":"1f31a275"},"/SynapseML/docs/1.0.2/Use with MLFlow/Overview/-e3c":{"__comp":"17896441","content":"55203621"},"/SynapseML/docs/1.0.3/-60e":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"66d63bfc"},"/SynapseML/docs/1.0.3/Deploy Models/Overview/-1cf":{"__comp":"17896441","content":"ddacd194"},"/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/-695":{"__comp":"17896441","content":"6e2678b7"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-0c3":{"__comp":"17896441","content":"fe5d19cb"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/-e88":{"__comp":"17896441","content":"321ba9c5"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-6d7":{"__comp":"17896441","content":"4b18e5ae"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/-0d9":{"__comp":"17896441","content":"d36f3429"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-5bc":{"__comp":"17896441","content":"2c5f0abd"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/-3ee":{"__comp":"17896441","content":"6e4e5cd9"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-7b8":{"__comp":"17896441","content":"51878b77"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-5f9":{"__comp":"17896441","content":"b35d004b"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-85d":{"__comp":"17896441","content":"d81d1137"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-6e8":{"__comp":"17896441","content":"2e93ae0f"},"/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-e1a":{"__comp":"17896441","content":"b739c60a"},"/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-54d":{"__comp":"17896441","content":"556ad472"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/-562":{"__comp":"17896441","content":"59d9fecc"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-734":{"__comp":"17896441","content":"5407376f"},"/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-410":{"__comp":"17896441","content":"3c22aa66"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-823":{"__comp":"17896441","content":"0a9c6347"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/-1ea":{"__comp":"17896441","content":"1293b2b0"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-96b":{"__comp":"17896441","content":"943afd32"},"/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-432":{"__comp":"17896441","content":"08019edc"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/-f57":{"__comp":"17896441","content":"01587ade"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/-141":{"__comp":"17896441","content":"f3e986ee"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/-005":{"__comp":"17896441","content":"328f3af4"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-2d6":{"__comp":"17896441","content":"888e2229"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-abe":{"__comp":"17896441","content":"3e7c5569"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-fb6":{"__comp":"17896441","content":"1a62d1af"},"/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-1fe":{"__comp":"17896441","content":"c4d68582"},"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-561":{"__comp":"17896441","content":"a563fa93"},"/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-158":{"__comp":"17896441","content":"cee4c3b9"},"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/-e1b":{"__comp":"17896441","content":"7ad384dc"},"/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-99d":{"__comp":"17896441","content":"5a3e1bca"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/-1f6":{"__comp":"17896441","content":"1518f45a"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/-b8d":{"__comp":"17896441","content":"c764f2c0"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-3b4":{"__comp":"17896441","content":"d7f58694"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-146":{"__comp":"17896441","content":"63cd8b22"},"/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/-57d":{"__comp":"17896441","content":"88f8e3e1"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/-e16":{"__comp":"17896441","content":"e03f5a54"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-3d8":{"__comp":"17896441","content":"c3dac62d"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-30f":{"__comp":"17896441","content":"3a7a88af"},"/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-e7e":{"__comp":"17896441","content":"fe6131e8"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/-8c5":{"__comp":"17896441","content":"b3272230"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/-1f6":{"__comp":"17896441","content":"75514f6a"},"/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-ea9":{"__comp":"17896441","content":"b66ab6b5"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/-211":{"__comp":"17896441","content":"5dbf4619"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/-18d":{"__comp":"17896441","content":"980b00ea"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/-d3d":{"__comp":"17896441","content":"f027bb15"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-025":{"__comp":"17896441","content":"de3bb7c3"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-878":{"__comp":"17896441","content":"16d8ca91"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-326":{"__comp":"17896441","content":"f5b4e517"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-a63":{"__comp":"17896441","content":"e5fe80e3"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/-8ed":{"__comp":"17896441","content":"d84904f9"},"/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/-893":{"__comp":"17896441","content":"455aa1bd"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-747":{"__comp":"17896441","content":"82d7cc79"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-484":{"__comp":"17896441","content":"3ea19ec0"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/-60f":{"__comp":"17896441","content":"7aa49360"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-c16":{"__comp":"17896441","content":"f537d1e9"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-35d":{"__comp":"17896441","content":"0713a501"},"/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-f74":{"__comp":"17896441","content":"4801d370"},"/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/-ed3":{"__comp":"17896441","content":"f228fdc5"},"/SynapseML/docs/1.0.3/Get Started/Install SynapseML/-df3":{"__comp":"17896441","content":"2d8a44d7"},"/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/-450":{"__comp":"17896441","content":"ac39f7dc"},"/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/-dff":{"__comp":"17896441","content":"83203dd1"},"/SynapseML/docs/1.0.3/Overview/-094":{"__comp":"17896441","content":"39a50b22"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/-505":{"__comp":"17896441","content":"cf666d49"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/-6ff":{"__comp":"17896441","content":"9688a5be"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/-104":{"__comp":"17896441","content":"faca745a"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/-08b":{"__comp":"17896441","content":"bc85b804"},"/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/-74d":{"__comp":"17896441","content":"10b7acf6"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/-baf":{"__comp":"17896441","content":"e402e26d"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/-dcc":{"__comp":"17896441","content":"75801678"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/-80b":{"__comp":"17896441","content":"4585b70e"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/-4af":{"__comp":"17896441","content":"c39e3602"},"/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/-9cd":{"__comp":"17896441","content":"604b2d50"},"/SynapseML/docs/1.0.3/Reference/Contributor Guide/-676":{"__comp":"17896441","content":"275c0efd"},"/SynapseML/docs/1.0.3/Reference/Developer Setup/-904":{"__comp":"17896441","content":"ed40d5f1"},"/SynapseML/docs/1.0.3/Reference/Docker Setup/-963":{"__comp":"17896441","content":"3ee83f58"},"/SynapseML/docs/1.0.3/Reference/Dotnet Setup/-c7f":{"__comp":"17896441","content":"09990b79"},"/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/-9f9":{"__comp":"17896441","content":"f0649d54"},"/SynapseML/docs/1.0.3/Reference/R Setup/-3a0":{"__comp":"17896441","content":"d57515f7"},"/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/-00b":{"__comp":"17896441","content":"b35e9998"},"/SynapseML/docs/1.0.3/Use with MLFlow/Install/-5a1":{"__comp":"17896441","content":"17f418b7"},"/SynapseML/docs/1.0.3/Use with MLFlow/Overview/-234":{"__comp":"17896441","content":"8a1bca9d"},"/SynapseML/docs/next/-3d8":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"935f2afb"},"/SynapseML/docs/next/Deploy Models/Overview/-29d":{"__comp":"17896441","content":"94d743d6"},"/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/-eeb":{"__comp":"17896441","content":"2c98ca3e"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-12a":{"__comp":"17896441","content":"0349e4f0"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/-00a":{"__comp":"17896441","content":"318ccb43"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-e9b":{"__comp":"17896441","content":"e464d112"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/-0d4":{"__comp":"17896441","content":"fa2e2688"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-6d0":{"__comp":"17896441","content":"0e3f998d"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/-bef":{"__comp":"17896441","content":"57ef17c5"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-4f8":{"__comp":"17896441","content":"9530a2cf"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-3a8":{"__comp":"17896441","content":"1b7af47d"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-385":{"__comp":"17896441","content":"c0391845"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5c7":{"__comp":"17896441","content":"97592aac"},"/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-f4f":{"__comp":"17896441","content":"0a954229"},"/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-bf9":{"__comp":"17896441","content":"b4a95996"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/-89c":{"__comp":"17896441","content":"b83021be"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-fc9":{"__comp":"17896441","content":"d707d2ed"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-80d":{"__comp":"17896441","content":"0fb98d33"},"/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/-b09":{"__comp":"17896441","content":"58b8f176"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-12f":{"__comp":"17896441","content":"3bed3e2f"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/-86b":{"__comp":"17896441","content":"41f327fc"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-25e":{"__comp":"17896441","content":"2bcda99e"},"/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-c31":{"__comp":"17896441","content":"1e14124f"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/-1a6":{"__comp":"17896441","content":"5e7c3303"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/-270":{"__comp":"17896441","content":"42736d5f"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/-c9a":{"__comp":"17896441","content":"ea2d2bcc"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-fdc":{"__comp":"17896441","content":"6a07b55e"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-9cf":{"__comp":"17896441","content":"90e4432b"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-a7b":{"__comp":"17896441","content":"5eddefdd"},"/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-641":{"__comp":"17896441","content":"bd1bcb2c"},"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-6e0":{"__comp":"17896441","content":"40b49758"},"/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-8d1":{"__comp":"17896441","content":"905ce8b5"},"/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/-50f":{"__comp":"17896441","content":"1d820955"},"/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-d9a":{"__comp":"17896441","content":"5225b7e0"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/-ef5":{"__comp":"17896441","content":"96cf5ff0"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/-43a":{"__comp":"17896441","content":"ed6d544d"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/-a4b":{"__comp":"17896441","content":"dd3f9ada"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-a86":{"__comp":"17896441","content":"52880d18"},"/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-101":{"__comp":"17896441","content":"120352d6"},"/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/-5d7":{"__comp":"17896441","content":"83586bff"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/-1e7":{"__comp":"17896441","content":"d8b3ac6a"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-88e":{"__comp":"17896441","content":"9c279ae7"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-d5d":{"__comp":"17896441","content":"9da9112f"},"/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-41e":{"__comp":"17896441","content":"91e9cb67"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/-d73":{"__comp":"17896441","content":"2e5f4263"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/-267":{"__comp":"17896441","content":"31bb7274"},"/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-95b":{"__comp":"17896441","content":"66fcb786"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/-798":{"__comp":"17896441","content":"69dc6d0a"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/-0ec":{"__comp":"17896441","content":"51dbb02f"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/-039":{"__comp":"17896441","content":"c5c95e45"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-2fd":{"__comp":"17896441","content":"2137a7cd"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-8f3":{"__comp":"17896441","content":"b5375b6f"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-484":{"__comp":"17896441","content":"2406587d"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-3d2":{"__comp":"17896441","content":"bc6a9944"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/-caf":{"__comp":"17896441","content":"48cf7354"},"/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/-219":{"__comp":"17896441","content":"d13e532e"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-b35":{"__comp":"17896441","content":"a81cf8b0"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-c0f":{"__comp":"17896441","content":"2d527871"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/-fae":{"__comp":"17896441","content":"610d430a"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-3c9":{"__comp":"17896441","content":"33b3776b"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-71e":{"__comp":"17896441","content":"41ee0cff"},"/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-e9d":{"__comp":"17896441","content":"8eb438b4"},"/SynapseML/docs/next/Get Started/Create a Spark Cluster/-37f":{"__comp":"17896441","content":"a19ce767"},"/SynapseML/docs/next/Get Started/Install SynapseML/-975":{"__comp":"17896441","content":"73c2022c"},"/SynapseML/docs/next/Get Started/Quickstart - Your First Models/-8a6":{"__comp":"17896441","content":"283b8f99"},"/SynapseML/docs/next/Get Started/Set up Cognitive Services/-47d":{"__comp":"17896441","content":"b2be3901"},"/SynapseML/docs/next/Overview/-9a5":{"__comp":"17896441","content":"cbc3190d"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/-c47":{"__comp":"17896441","content":"521f9727"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/-6e1":{"__comp":"17896441","content":"34ff21b6"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_core/-7e6":{"__comp":"17896441","content":"9783ff24"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/-5eb":{"__comp":"17896441","content":"792eae7e"},"/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/-49f":{"__comp":"17896441","content":"88e44e99"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/-443":{"__comp":"17896441","content":"1b3ab3b8"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_core/-0a9":{"__comp":"17896441","content":"bb7a48da"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/-137":{"__comp":"17896441","content":"f7a32432"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/-3f4":{"__comp":"17896441","content":"69bb4b11"},"/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/-d93":{"__comp":"17896441","content":"8590feaa"},"/SynapseML/docs/next/Reference/Contributor Guide/-dec":{"__comp":"17896441","content":"104bf0a6"},"/SynapseML/docs/next/Reference/Developer Setup/-1c5":{"__comp":"17896441","content":"1db1f785"},"/SynapseML/docs/next/Reference/Docker Setup/-598":{"__comp":"17896441","content":"645abc7d"},"/SynapseML/docs/next/Reference/Dotnet Setup/-ded":{"__comp":"17896441","content":"f85db098"},"/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/-8a8":{"__comp":"17896441","content":"5b779334"},"/SynapseML/docs/next/Reference/R Setup/-d8b":{"__comp":"17896441","content":"44ff0d07"},"/SynapseML/docs/next/Use with MLFlow/Autologging/-0ca":{"__comp":"17896441","content":"c95882d3"},"/SynapseML/docs/next/Use with MLFlow/Install/-9b7":{"__comp":"17896441","content":"fbb011bb"},"/SynapseML/docs/next/Use with MLFlow/Overview/-b10":{"__comp":"17896441","content":"a6191053"},"/SynapseML/docs/-352":{"__comp":"1be78505","__context":{"plugin":"d8d6ba90"},"versionMetadata":"76bf568a"},"/SynapseML/docs/Deploy Models/Overview/-438":{"__comp":"17896441","content":"3b75a1c6"},"/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/-e9f":{"__comp":"17896441","content":"3d963cd5"},"/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/-3f1":{"__comp":"17896441","content":"4f81391a"},"/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/-c3c":{"__comp":"17896441","content":"d05981ec"},"/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/-de7":{"__comp":"17896441","content":"56543a15"},"/SynapseML/docs/Explore Algorithms/AI Services/Overview/-52c":{"__comp":"17896441","content":"8bff2444"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/-dc7":{"__comp":"17896441","content":"50f80512"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/-2c5":{"__comp":"17896441","content":"a1824f78"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/-f90":{"__comp":"17896441","content":"3dafa002"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/-425":{"__comp":"17896441","content":"0eaf8468"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/-d37":{"__comp":"17896441","content":"dc7f7ac1"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/-5e1":{"__comp":"17896441","content":"1766722a"},"/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/-eb1":{"__comp":"17896441","content":"8a46fce0"},"/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/-7e3":{"__comp":"17896441","content":"efbb59fb"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/-75a":{"__comp":"17896441","content":"23af8e3e"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/-b1b":{"__comp":"17896441","content":"e863531f"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/-cf3":{"__comp":"17896441","content":"8d990105"},"/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/-bcf":{"__comp":"17896441","content":"7aeb6f16"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/-3e2":{"__comp":"17896441","content":"7296bed0"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/-618":{"__comp":"17896441","content":"368de161"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/-7c2":{"__comp":"17896441","content":"27de0303"},"/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/-a7a":{"__comp":"17896441","content":"3ef20ed6"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/-1ba":{"__comp":"17896441","content":"e0077969"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/-628":{"__comp":"17896441","content":"1f6f5ac9"},"/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/-90b":{"__comp":"17896441","content":"59e4d88e"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/-57b":{"__comp":"17896441","content":"8799ef55"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/-b77":{"__comp":"17896441","content":"6094981e"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/-a81":{"__comp":"17896441","content":"38b375cc"},"/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/-93c":{"__comp":"17896441","content":"78cb5959"},"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/-87a":{"__comp":"17896441","content":"a5ffe972"},"/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/-699":{"__comp":"17896441","content":"b4ae214b"},"/SynapseML/docs/Explore Algorithms/LightGBM/Overview/-300":{"__comp":"17896441","content":"141dc4a8"},"/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/-5c1":{"__comp":"17896441","content":"2fb6c1b0"},"/SynapseML/docs/Explore Algorithms/OpenAI/-06d":{"__comp":"17896441","content":"d1428557"},"/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/-3b0":{"__comp":"17896441","content":"2461af4d"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/-b8b":{"__comp":"17896441","content":"c76e8cac"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/-d6d":{"__comp":"17896441","content":"c82134d8"},"/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/-baa":{"__comp":"17896441","content":"7275f11a"},"/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/-179":{"__comp":"17896441","content":"d95a70ea"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/-4b2":{"__comp":"17896441","content":"2cec6bcf"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/-54a":{"__comp":"17896441","content":"f264933a"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/-572":{"__comp":"17896441","content":"eed630e0"},"/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/-743":{"__comp":"17896441","content":"6c82e982"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/-82e":{"__comp":"17896441","content":"bf04857e"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/-dc2":{"__comp":"17896441","content":"affaa541"},"/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/-72f":{"__comp":"17896441","content":"eb0ef952"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/-e8d":{"__comp":"17896441","content":"014ce9a3"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/-e6c":{"__comp":"17896441","content":"ce7af6ae"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/-f44":{"__comp":"17896441","content":"3bbe0f34"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/-9d9":{"__comp":"17896441","content":"26631026"},"/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/-082":{"__comp":"17896441","content":"5e008bd1"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/-f62":{"__comp":"17896441","content":"b92ffdc6"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/-804":{"__comp":"17896441","content":"5efd5006"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/-60b":{"__comp":"17896441","content":"8ff6772d"},"/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/-3c5":{"__comp":"17896441","content":"644ff257"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/-32b":{"__comp":"17896441","content":"dfb6dfe9"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/-046":{"__comp":"17896441","content":"b7226801"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/-5b3":{"__comp":"17896441","content":"fc598b5d"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/-490":{"__comp":"17896441","content":"56087ddb"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/-5dd":{"__comp":"17896441","content":"01eb996b"},"/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/-804":{"__comp":"17896441","content":"fa889bee"},"/SynapseML/docs/Get Started/Create a Spark Cluster/-37d":{"__comp":"17896441","content":"cc52e3cd"},"/SynapseML/docs/Get Started/Install SynapseML/-16c":{"__comp":"17896441","content":"62ccfbde"},"/SynapseML/docs/Get Started/Quickstart - Your First Models/-e0c":{"__comp":"17896441","content":"e84755e1"},"/SynapseML/docs/Get Started/Set up Cognitive Services/-ea8":{"__comp":"17896441","content":"ce40236b"},"/SynapseML/docs/Overview/-58e":{"__comp":"17896441","content":"bf8a5c8a"},"/SynapseML/docs/Quick Examples/estimators/estimators_causal/-e9d":{"__comp":"17896441","content":"104b9b56"},"/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/-419":{"__comp":"17896441","content":"79d79762"},"/SynapseML/docs/Quick Examples/estimators/estimators_core/-63c":{"__comp":"17896441","content":"421d486c"},"/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/-d43":{"__comp":"17896441","content":"b28fe732"},"/SynapseML/docs/Quick Examples/estimators/estimators_vw/-bd9":{"__comp":"17896441","content":"1ef65056"},"/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/-bbd":{"__comp":"17896441","content":"408524d4"},"/SynapseML/docs/Quick Examples/transformers/transformers_core/-1a1":{"__comp":"17896441","content":"5bf420e9"},"/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/-483":{"__comp":"17896441","content":"5c0b440a"},"/SynapseML/docs/Quick Examples/transformers/transformers_opencv/-8a0":{"__comp":"17896441","content":"513214d5"},"/SynapseML/docs/Quick Examples/transformers/transformers_vw/-fd7":{"__comp":"17896441","content":"21f47424"},"/SynapseML/docs/Reference/Contributor Guide/-6e9":{"__comp":"17896441","content":"294c060e"},"/SynapseML/docs/Reference/Developer Setup/-367":{"__comp":"17896441","content":"b88f1f9b"},"/SynapseML/docs/Reference/Docker Setup/-a2a":{"__comp":"17896441","content":"405fabd0"},"/SynapseML/docs/Reference/Dotnet Setup/-995":{"__comp":"17896441","content":"58486a98"},"/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/-e58":{"__comp":"17896441","content":"4631a032"},"/SynapseML/docs/Reference/R Setup/-d4d":{"__comp":"17896441","content":"04804d02"},"/SynapseML/docs/Use with MLFlow/Autologging/-1cd":{"__comp":"17896441","content":"2a023663"},"/SynapseML/docs/Use with MLFlow/Install/-a7e":{"__comp":"17896441","content":"70e8008e"},"/SynapseML/docs/Use with MLFlow/Overview/-424":{"__comp":"17896441","content":"15516451"},"/SynapseML/-6e1":{"__comp":"c4f5d8e4","__context":{"plugin":"627aefca"},"config":"5e9f5e1a"}}')}},e=>{e.O(0,[40532],(()=>{return t=49383,e(e.s=t);var t}));e.O()}]);
\ No newline at end of file
diff --git a/assets/js/runtime~main.6b063ea3.js b/assets/js/runtime~main.6b063ea3.js
new file mode 100644
index 0000000000..f08717b638
--- /dev/null
+++ b/assets/js/runtime~main.6b063ea3.js
@@ -0,0 +1 @@
+(()=>{"use strict";var e,b,f,d,a,c={},t={};function r(e){var b=t[e];if(void 0!==b)return b.exports;var f=t[e]={exports:{}};return c[e].call(f.exports,f,f.exports,r),f.exports}r.m=c,e=[],r.O=(b,f,d,a)=>{if(!f){var c=1/0;for(i=0;i<e.length;i++){f=e[i][0],d=e[i][1],a=e[i][2];for(var t=!0,o=0;o<f.length;o++)(!1&a||c>=a)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,a<c&&(c=a));if(t){e.splice(i--,1);var n=d();void 0!==n&&(b=n)}}return b}a=a||0;for(var i=e.length;i>0&&e[i-1][2]>a;i--)e[i]=e[i-1];e[i]=[f,d,a]},r.n=e=>{var b=e&&e.__esModule?()=>e.default:()=>e;return r.d(b,{a:b}),b},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,d){if(1&d&&(e=this(e)),8&d)return e;if("object"==typeof e&&e){if(4&d&&e.__esModule)return e;if(16&d&&"function"==typeof e.then)return e}var a=Object.create(null);r.r(a);var c={};b=b||[null,f({}),f([]),f(f)];for(var t=2&d&&e;"object"==typeof t&&!~b.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((b=>c[b]=()=>e[b]));return c.default=()=>e,r.d(a,c),a},r.d=(e,b)=>{for(var f in b)r.o(b,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:b[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((b,f)=>(r.f[f](e,b),b)),[])),r.u=e=>"assets/js/"+({226:"32fe2e34",271:"e8d66fc1",360:"88b63415",390:"001aea4c",403:"7b60d8e3",429:"c4d68582",587:"b83021be",688:"b66ab6b5",787:"20641120",806:"b3272230",847:"73c2022c",939:"0a65b9de",1084:"421d486c",1123:"1f6f5ac9",1172:"255635fc",1283:"f264933a",1286:"fd4bd09d",1773:"3cbfbf75",1802:"513002de",1854:"ad10988d",1985:"368de161",2070:"6c82e982",2195:"b72abe57",2381:"59e4d88e",2408:"09bc1516",2797:"e4e95844",2901:"980b00ea",3202:"6c61f93b",3554:"55203621",3611:"6e159789",3701:"513214d5",3765:"d1428557",3814:"b41f295c",3871:"445012ce",3953:"60a2189a",4026:"73cf52e8",4193:"1ad91055",4204:"ea2d2bcc",4503:"1e0d2d3b",4800:"c065fcb9",5e3:"cf57716c",5253:"405fabd0",5254:"33b3776b",5287:"b8963dc0",5378:"fc338b02",5789:"1155dc6f",5909:"78cb5959",6004:"10b7acf6",6289:"75801678",6304:"5e008bd1",6413:"4bbbdfcf",6633:"707d2a35",6712:"ed40d5f1",7201:"567cd2fd",7234:"268b7b40",7509:"9530a2cf",7925:"9c5088a2",8006:"9da9112f",8040:"e24ee294",8146:"d13e532e",8168:"e7bd83ca",8749:"2d527871",8839:"a563fa93",8966:"41f327fc",9296:"1518f45a",9540:"4f1ca6a9",10643:"014ce9a3",10912:"e12b4691",10978:"ac581902",11298:"c76e8cac",11542:"fba72e87",11561:"2ece0fbe",11977:"3b5de274",12124:"d05981ec",12398:"6c7ccb6d",12630:"5962ef6e",12824:"75514f6a",12840:"4bea8531",13431:"1b3ab3b8",13482:"ef6ec597",14273:"1b7af47d",14341:"a5ffe972",14535:"1842239f",15095:"90e4432b",15149:"603706f6",15165:"4f81391a",15253:"b7787e72",15343:"1abf1e18",15404:"ac39f7dc",15487:"3a4f9d93",15511:"a9330763",15566:"ebf648dd",15579:"dd29718b",15788:"f3e986ee",15800:"18c7d2a3",15839:"07f6b90a",16030:"0f113696",16269:"76d23901",16286:"bef0d2d3",16551:"3c1f4383",16833:"5b779334",16975:"51dbb02f",17026:"3ea19ec0",17295:"6eddee4c",17538:"ad139e3c",17542:"66d63bfc",17655:"b1706689",17782:"b328a361",17817:"1766722a",17820:"dd3f9ada",17973:"63cd8b22",18373:"445a1b28",18443:"ceeda143",18563:"d84904f9",18825:"c73cfc52",18828:"514f485e",18933:"11c317cb",19042:"c14fdd92",19336:"8bff2444",19465:"548964ce",19483:"15ea8379",19603:"2b30bc14",19775:"b2be3901",19950:"275c0efd",21102:"20e0fe38",21401:"6e4e5cd9",21933:"cb7c2a83",22007:"a34c49d0",22011:"e316de39",22287:"5407376f",22341:"aa5548ab",22577:"8af72580",22707:"49e6864b",22712:"6b40ec54",22970:"88f8e3e1",23036:"052105dd",23234:"c0391845",23544:"4e3910af",23807:"f21c13c4",23838:"92ce4fd5",23894:"35826f14",23941:"2bcda99e",23979:"48329b94",24046:"7aeb6f16",24422:"c5c95e45",24484:"1a3608a9",24594:"b962e007",24627:"e579bad6",25116:"10e13725",25449:"cf666d49",25844:"cee4c3b9",25990:"38d6824e",26021:"295a8e78",26184:"0eaf8468",26676:"03d0d544",26685:"c2e57a1b",26871:"8a1bca9d",27497:"ce40236b",27572:"a07d09c1",27813:"35bdc661",27918:"17896441",27991:"dc1e40d7",28456:"8d990105",28465:"6e2678b7",28559:"dac82c00",28634:"f1b2dc7f",28696:"b066233f",28746:"d620a661",28955:"6e71bda3",28957:"2d5b0e6a",29006:"9783ff24",29283:"7c007ec6",29514:"1be78505",29535:"905ce8b5",29540:"c82134d8",29696:"166d99f5",29806:"b9f125cc",29845:"39a50b22",30013:"5eddefdd",30023:"b739c60a",30301:"627aefca",30335:"22444eb9",30460:"0a9c6347",30464:"47bbce76",30927:"a6191053",31153:"c95882d3",31279:"fd3b6ea5",31333:"521f9727",31836:"645abc7d",31896:"9fa7e8eb",32185:"561bd03d",32599:"5f17911b",32777:"8ff6772d",32789:"0606f1a5",32855:"5bf420e9",33064:"da508cef",33195:"6a26e359",33358:"5c18deb5",33631:"90f00051",33988:"01587ade",34334:"141dc4a8",34397:"3e1e84da",34654:"e887b273",34797:"0d432c84",34940:"7275f11a",35015:"2c5f0abd",35024:"0daa5b3f",35040:"767a7177",35056:"c3b1d949",35124:"061b38c5",35328:"54287b47",35517:"22f921cc",35537:"c3dac62d",35750:"0637d1e7",35939:"0453ae68",36005:"90742975",36204:"50af03e4",36224:"6c7d288d",36285:"b0e28723",36572:"589adaf2",36585:"b7226801",36673:"affaa541",36698:"aba932dd",36746:"5ef28062",36773:"b4ae214b",36828:"df01388f",36916:"f027bb15",36958:"1e57c92a",37001:"83586bff",37069:"40b49758",37334:"27de0303",37392:"ef77ef39",37474:"21f47424",37486:"a406e231",37592:"0713a501",37610:"077ca2ec",37638:"d57515f7",37676:"4631a032",37742:"5efd5006",38057:"d9e5e528",38078:"6f8cd013",38261:"012c1c0f",38377:"70b306ba",38399:"327be84b",38759:"16d05917",38781:"5e7c3303",38846:"bf04857e",38859:"b7802ae1",38962:"4b18e5ae",39139:"ae89cd3b",39242:"bb7a48da",39307:"0d6d64cd",39402:"16d8ca91",39596:"4e105cba",40023:"2e93ae0f",40367:"fece3c24",40412:"9e5b788f",40659:"610d430a",40852:"965ed185",40950:"1db1f785",41607:"04804d02",41719:"b66e2100",41867:"391cb159",41944:"79114dfe",41987:"2cec6bcf",42037:"328f3af4",42283:"e3089280",42377:"1789daeb",42428:"31ee0fa3",42522:"6c4c8509",42638:"e21a3367",42802:"3ee83f58",42815:"993c5022",42852:"423540f2",42947:"42736d5f",42970:"de3bb7c3",43156:"1587ba1d",43251:"455aa1bd",43344:"995576e9",43426:"494501bd",43488:"336404dc",43730:"034e6169",44177:"d41234a3",44178:"70fe7dd7",44334:"00ab2cda",44422:"cd8e97a5",44490:"d81d1137",44504:"99e5ffa1",44534:"2d8a44d7",44539:"fe6131e8",45282:"104b9b56",45370:"7649ec47",45503:"bc85b804",45589:"7ad384dc",45938:"8799ef55",46103:"ccc49370",46135:"96cf5ff0",46338:"0fb98d33",46559:"98258c5e",46613:"b4898d44",46807:"c39e3602",46845:"79d79762",47454:"c5775233",47552:"01eb996b",48007:"408524d4",48491:"59ef8022",48602:"25fb872c",48807:"13246fd5",49101:"eed630e0",49196:"efbc69e1",49778:"3e7c5569",49814:"3a7a88af",50002:"fc598b5d",50040:"a81cf8b0",50196:"9c279ae7",50297:"fe2c893b",50653:"97592aac",50745:"069e3c4c",50783:"5c0b440a",51078:"ff428354",51105:"6b9bdd6f",51317:"6a9ad78d",51478:"b4a95996",51732:"e9aab890",51842:"d8d6ba90",51910:"644ff257",51918:"9688a5be",52406:"c4d09a44",52433:"345903d4",52447:"2e5f4263",52535:"814f3328",52730:"a3fd5a6d",52752:"9b020c4a",52787:"ed5c6c48",52804:"12b89d4a",53017:"bf002efe",53050:"943afd32",53090:"86ed1ff2",53338:"b88f1f9b",53572:"a8456d9f",53608:"9e4087bc",54528:"cb5f1a06",54890:"f537d1e9",55276:"76b137bf",55305:"ca0a1bad",55538:"bece3771",55702:"41ee0cff",55864:"ce7af6ae",56373:"cff2e56e",57074:"1293b2b0",57168:"d9837698",57231:"9d095dba",57258:"e2689dfc",57689:"39122aab",57887:"be95fd8f",57924:"15a59c33",57930:"d39aa6d3",58092:"4d31bfb3",58462:"d989c707",58524:"0349e4f0",58738:"b940e0b5",58755:"990f07de",58846:"3bed3e2f",58861:"7c2e1100",59076:"dfb6dfe9",59394:"4801d370",59484:"71042800",59823:"76bf568a",60055:"403e642d",60445:"f322d2da",60880:"f0649d54",61407:"3ef20ed6",61428:"b5375b6f",61442:"b75118f0",61755:"04fc65cc",61778:"a0d8a1a2",61829:"f38a0cbd",62079:"5cfa133c",62106:"2c98ca3e",62217:"04ee4ab4",62465:"ddacd194",62565:"c38c658c",62675:"4ef2f7cf",62721:"d95a70ea",62750:"52880d18",62773:"08019edc",63002:"bf8a5c8a",63099:"95d9e891",63196:"74091a62",63402:"56543a15",63448:"ed6d544d",64172:"3bbe0f34",64195:"c4f5d8e4",64233:"b7c6295c",64482:"0e3f998d",64713:"d355047f",64927:"fa2e2688",64992:"37ab9beb",65437:"44ff0d07",65752:"c3c516ff",65942:"900b935c",66167:"74659d33",66419:"e5fe80e3",66555:"0fa36e32",66808:"8462374b",66945:"296054a7",66981:"e863531f",67218:"ba9c0924",67287:"99f8fee5",67394:"31bb7274",67448:"888e2229",67597:"a608660b",67664:"faca745a",67706:"db170a23",67870:"ed316aed",68070:"a1824f78",68283:"e84755e1",68314:"104bf0a6",68526:"3201258e",68549:"c991f47b",68585:"18c9c2ba",68809:"fd748117",69159:"57e687e8",69226:"8181e18a",69314:"232d09b0",69559:"3fb29942",69966:"d1246f49",70224:"77c259dd",70344:"7aa7b8e0",70406:"6a07b55e",70880:"2461af4d",71535:"1a215693",71580:"179d57fa",72141:"a877f9e5",72249:"38b375cc",72286:"3d963cd5",72293:"56087ddb",72326:"1ef65056",72912:"13f17286",73060:"75406112",73249:"5225b7e0",73419:"8e32e44d",73443:"91e9cb67",73602:"af1e70b6",73796:"6a95f87e",74275:"792eae7e",74772:"4997ef4f",74963:"6f479459",75088:"7a1d9beb",75488:"cbc3190d",75586:"e7250a93",75658:"c23b5e26",76258:"3dafa002",76300:"d252041b",76454:"34f00221",76515:"9cb6fa7c",76572:"58486a98",76843:"62983deb",77225:"51878b77",77390:"b35e9998",77491:"d3093636",77682:"50f80512",77697:"39582d99",77969:"5887dc62",78047:"3b75a1c6",78480:"26631026",78579:"0b416bde",78817:"88e44e99",79043:"00353f68",79079:"efc1978d",79088:"0a954229",79328:"a0434473",79362:"34ff21b6",80053:"935f2afb",80112:"8fd0c721",80322:"b576c645",80598:"e1269457",80707:"ab9a8589",80751:"556ad472",80829:"ee6cea3b",80837:"17f418b7",80874:"b92ffdc6",80982:"2fb6c1b0",81102:"d25cde70",81340:"e1b2b5f3",81663:"00056db7",81787:"e4ba4487",81878:"fa889bee",81913:"1d820955",81970:"01a15f20",82158:"f7580424",82412:"12d85ab5",82569:"e03f5a54",82643:"f228fdc5",82668:"3a1147b5",82681:"62ccfbde",82723:"23af8e3e",82744:"f8597cb0",82880:"e1ed1e14",82994:"48cf7354",83251:"c34b1e36",83378:"fbb011bb",83393:"3e9a7422",83669:"d707d2ed",83781:"866b91a1",83790:"604b2d50",84202:"c2492d56",84207:"f14633e7",84298:"6094981e",84880:"f8aab3ca",84908:"59d9fecc",84954:"f7a32432",84979:"e8b9c2c7",85043:"8eb438b4",85122:"c2e40c34",85332:"321ba9c5",85433:"ffb7f88a",85648:"ee83122d",85729:"56d039a1",85847:"4e044b4e",86184:"4130db01",86248:"7bf16cc6",86482:"3c22aa66",86704:"67e06a8b",86856:"b80b493a",87320:"c40984d1",87384:"b35d004b",87452:"93ba15e0",87597:"66fcb786",87684:"b38eec89",87808:"b28fe732",88003:"5066efb2",88838:"70e8008e",88984:"d535b358",89220:"a2b5e5df",89262:"120352d6",89407:"deef0af8",89450:"83203dd1",89461:"6ff7775f",89463:"a19ce767",89767:"c413b43a",89864:"1dc2f362",90288:"69bb4b11",90544:"4585b70e",91078:"80472ecc",91103:"3ac37a06",91210:"e2ce57ed",91214:"cc52e3cd",91387:"d36f3429",91391:"2137a7cd",91570:"4de42975",91747:"85aed2f9",91755:"bc4bf151",91824:"c90c942b",91871:"8590feaa",91895:"78aebd5d",91921:"8328d740",92018:"dc7f7ac1",92045:"0e0ee9b2",92059:"b33598ac",92321:"325e22bd",92805:"d175df5e",92836:"8a46fce0",92867:"5dbf4619",92982:"3647ac9b",93089:"a6aa9e1f",93377:"318ccb43",94330:"d8b3ac6a",94353:"1f31a275",94395:"0e2bf69e",94508:"e464d112",94548:"53a5cb1e",94553:"d7f58694",94579:"94d743d6",94597:"d0cb74df",94603:"f7bfff73",94623:"e402e26d",94702:"5a3e1bca",94744:"1014102e",94756:"4cee39d7",94916:"82878c84",94994:"109f6864",95049:"f85db098",95064:"e9b75d67",95521:"283b8f99",95584:"bd1bcb2c",95602:"57ef17c5",95717:"d0c5e4fa",95768:"a34b09d4",95974:"8cb11643",96350:"1e14124f",96673:"e5c4ef01",96816:"82d7cc79",96922:"00e0a48e",96960:"1a62d1af",97030:"c764f2c0",97065:"fe5d19cb",97165:"294c060e",97297:"e0077969",97381:"4db82253",97715:"2a023663",97721:"0e16e9ea",97800:"09990b79",97920:"1a4e3797",97988:"4e432063",98050:"d3c9b0d7",98197:"eb0ef952",98462:"bc6a9944",98528:"7296bed0",98546:"f5b4e517",98583:"6bdbf6b8",99119:"2406587d",99128:"15516451",99147:"7aa49360",99161:"69dc6d0a",99374:"58b8f176",99378:"efbb59fb",99601:"fc47fc53",99781:"dea122e2",99970:"093b81f4"}[e]||e)+"."+{226:"7c60e348",271:"b472ae37",360:"0c049a9a",390:"16907556",403:"4c6e2b65",429:"c530f0a0",587:"25e7c11f",688:"feefe5ad",787:"4d89b7a8",806:"3689025c",847:"c07433f0",939:"dd4545ed",1084:"0194791b",1123:"640e13d4",1172:"105f1ab1",1283:"7a0aa43b",1286:"6ad5a370",1773:"3a54b67d",1802:"613317d6",1854:"c62a83e1",1985:"a23ad2e8",2070:"1b8bc6df",2195:"a42749a0",2381:"a9df8fd6",2408:"f68bf61a",2797:"ec2ce8dc",2901:"f0462d81",3202:"83225189",3554:"c9c5c25c",3611:"84cee5cb",3701:"ebe9d132",3765:"c7b94ddd",3814:"a51776da",3871:"3d351853",3953:"aa3ad659",4026:"91ffb89c",4193:"0c39c6fd",4204:"2942d62d",4503:"7ec496a3",4800:"f4fea7cb",5e3:"c4a07252",5253:"58f5ca42",5254:"d9a92894",5287:"b75edc26",5378:"be4cc3ff",5789:"294730b0",5909:"2b553671",6004:"fbfda01a",6289:"7e9ad05f",6304:"e1bb2c7e",6413:"406cd688",6633:"eeed6b75",6712:"11e439bd",7201:"97d9fe39",7234:"61faaca9",7509:"e0320a5b",7925:"45be218c",8006:"696da333",8040:"24249473",8146:"e8787811",8168:"805ffacc",8749:"fd6620c4",8839:"ef3c8cef",8966:"d111deb6",9296:"415afda5",9540:"fd3585d8",10643:"9ac2966d",10912:"5766341c",10978:"fa5141d9",11298:"a8bed199",11542:"4e543c37",11561:"4f426cb6",11977:"4f629e6e",12124:"92c4c2bb",12398:"ab17108f",12630:"89d0fd21",12824:"deab964d",12840:"15a46ad4",13431:"5b3464d7",13482:"f117cd47",14273:"ed1b0a2e",14341:"5f31f2f7",14535:"e5b29187",15095:"1fe4ec82",15149:"b1d50f20",15165:"9fc5aa61",15253:"eb95bb49",15343:"b4f9c1d7",15404:"d4e54d08",15487:"e356abfe",15511:"8de44ef4",15566:"7af2a42d",15579:"80d18a93",15788:"aaa06e2d",15800:"0daa1137",15839:"f15dc558",16030:"a1d2c595",16269:"2a5ae6a5",16286:"3091a1cb",16551:"8334c80a",16833:"ceeed8df",16975:"25f84d8d",17026:"40a33067",17295:"e7002340",17538:"68b2fa1b",17542:"b345b319",17655:"caf41e88",17782:"9af38353",17817:"7a60488d",17820:"97671d28",17973:"49a55e91",18373:"3b52ab84",18443:"8ed83508",18563:"a650d3a8",18825:"edd2110a",18828:"61dff53a",18894:"8ba79e33",18933:"82ff982d",19042:"f0e3a658",19336:"7f5c5c75",19465:"ba302a23",19483:"a04abc04",19603:"63140d8c",19775:"53d720b7",19950:"8e6a0524",21102:"e64026a7",21401:"25b8730b",21933:"284bd8a5",22007:"3c7d5195",22011:"12fd87b1",22287:"c1ff4071",22341:"a7c000a6",22577:"12ecb1e2",22707:"969377ed",22712:"204c8f47",22970:"02f3aa52",23036:"ba928913",23234:"6de1ab83",23544:"a2c9c406",23807:"f8ab332b",23838:"f96fd2ce",23894:"4a36373d",23941:"7eb4157a",23979:"da7d92d4",24046:"c95f1e59",24422:"14588453",24484:"290578f8",24572:"7376a9ae",24594:"aa995eb4",24627:"ad35e092",25116:"cadea3bd",25449:"28595d1a",25844:"67b2a5fc",25990:"d1dad3f6",26021:"6b9ce1ea",26184:"d1fb0dee",26676:"e6e7e7a4",26685:"04f76c9c",26871:"25fabaa5",27497:"8be44fac",27572:"92e50813",27813:"7670b946",27918:"1dd7c1ea",27991:"5af43466",28456:"42e1f8bb",28465:"bae03a70",28559:"53d784db",28634:"07a76afe",28696:"03d2fb68",28746:"258f03a6",28955:"3d9432bb",28957:"5c5fc2a0",29006:"bcf81dc8",29283:"a12f8681",29514:"216cf5d7",29535:"75ea1aee",29540:"3ae661ae",29696:"09a48be2",29806:"e13efecc",29845:"6229c1ae",30013:"9156458e",30023:"f70eac19",30301:"c4bbb92d",30335:"ebddc206",30460:"c79f944d",30464:"cfaedc94",30927:"5f7ddb44",31153:"c74b2845",31279:"771e833b",31333:"714da6d3",31836:"6afbe490",31896:"cce4e3d2",32185:"242c68ad",32599:"6ee262cb",32777:"9a4fbe82",32789:"d5b89e35",32855:"ab34218c",33064:"2b15599a",33195:"23458ac7",33358:"6875ed7c",33631:"a5a75b4b",33988:"0caae640",34334:"3a1990ef",34397:"50e63b4b",34654:"96dfebcb",34797:"cf49ef0b",34940:"7bb97a11",35015:"90bcb885",35024:"77f9b663",35040:"ba559dd9",35056:"6d5d38eb",35124:"6cc60e98",35328:"2329ba8e",35517:"5666354f",35537:"27d90dc8",35750:"064379cf",35939:"4fca9411",36005:"07beb68e",36204:"1b5a72e3",36224:"169580d7",36285:"770abe5c",36572:"3b4a629e",36585:"1717b0cf",36673:"6015f67e",36698:"24aeb102",36746:"b6d58204",36773:"a3eb6850",36828:"8ab9b559",36916:"627e234a",36958:"2b54b1f9",37001:"d663eac3",37069:"a5610a85",37334:"bd502a3e",37392:"3674eb36",37474:"8fc13a5a",37486:"be80e33d",37592:"84e46938",37610:"5ca2e1ca",37638:"6fbb6657",37676:"98d2551a",37742:"a5e70451",37853:"6b0071e3",38057:"0aa70e90",38078:"eaa3b4f7",38261:"3e7aa5d5",38377:"bd09de84",38399:"4a04eae7",38759:"873ccf3d",38781:"2f3351a0",38846:"88bd3c18",38859:"7ccc674c",38962:"0236cee6",39139:"6f56c354",39242:"40e4acd1",39307:"5d7232c1",39402:"d6599b0a",39596:"511860eb",40023:"caff10d8",40367:"8325e1db",40412:"8ecdd605",40659:"c8eb504a",40852:"b29206c4",40950:"7be04e11",41607:"38ca02bf",41719:"53c08f2d",41867:"7db460cb",41944:"4ec7bb98",41987:"0609e7ff",42037:"ad3a366f",42283:"a34f8074",42377:"4d69dc8d",42428:"7e0359d6",42522:"096248d0",42638:"082230ce",42802:"91ab538d",42815:"971a7a1a",42852:"bd015a5e",42947:"173cf0a5",42970:"0ca4866d",43156:"6e858d37",43251:"002f23aa",43344:"a5f5a399",43426:"97ac4633",43488:"3bcd1836",43730:"a22706a2",44177:"a7c47f47",44178:"6168e9e1",44334:"7f2d19ca",44422:"e1c7affe",44490:"901cceab",44504:"b8b106d8",44534:"7f1b5fc9",44539:"558a2d88",45282:"1eae2722",45370:"0d930dbf",45503:"a72ce2a2",45589:"7f42a2b5",45938:"ee06af10",46048:"f55105b0",46103:"4043289d",46135:"a123dbdf",46338:"073f7a96",46559:"d0e09414",46613:"0384012a",46807:"62a2c61f",46845:"4deeece5",46945:"fd128107",47454:"2e3cc69a",47552:"96f02664",48007:"1f47865f",48491:"2a20a978",48602:"f679dc4f",48807:"d5a3f483",49101:"a98291dd",49196:"867a0f29",49778:"cd04be09",49814:"cbb3c3ac",50002:"0471b2c6",50040:"e195e324",50196:"cf79206b",50297:"4da5326f",50653:"288194f1",50745:"fc78265d",50783:"3c16a52b",51078:"23039dab",51105:"9273b6ea",51317:"ff17fe39",51478:"f6b57981",51732:"ea31977e",51842:"a88f573c",51910:"eff31342",51918:"0614f4d5",52406:"4d053caf",52433:"30590104",52447:"d06ddc63",52535:"b03595a3",52730:"7a6b4a14",52752:"355782e7",52787:"838e0402",52804:"eefbdfaa",53017:"9348a25f",53050:"6abd1a54",53090:"75338eaf",53140:"a29368cb",53338:"083cbf6d",53572:"7effd19a",53608:"72da96df",54528:"ff5b94ed",54890:"af1bc49c",55276:"facd650b",55305:"1e659f3b",55538:"81957284",55702:"24761a8e",55864:"16c1759c",56373:"f0ab3974",57074:"b14d0343",57168:"9a260310",57231:"f08ac08d",57258:"87fca17c",57689:"0759cd84",57887:"3c00aa5f",57924:"929e4f47",57930:"1e91e4cb",58092:"80d71d4e",58462:"ef15d1f1",58524:"b857b4f3",58738:"43f88bbc",58755:"70e7bcad",58846:"bbcec631",58861:"99c57d57",59076:"823e2f17",59394:"989a950f",59484:"edba3628",59823:"c4a7067d",60055:"fb65aa03",60445:"deb5d152",60880:"4828013e",61407:"8dcb13c6",61428:"d1a4d228",61442:"6214188f",61755:"85679afc",61778:"42efbba1",61829:"79c4438f",62079:"3d50e52d",62106:"b11711e1",62217:"f597b4a5",62465:"26755e65",62565:"7331c10f",62675:"0c1bfb62",62721:"a55c8bbb",62750:"e185638d",62773:"35944844",63002:"cce5fea8",63099:"11b155f6",63196:"3a5e22f4",63402:"9a4cd328",63448:"6d9cca38",64172:"85cd1357",64195:"13fd239b",64233:"871b4ef9",64482:"d6d86e59",64713:"ce848f45",64927:"082fe1c7",64992:"2852555a",65437:"08ba8e6c",65752:"d9a972a6",65942:"3ac80a08",66167:"22cd1bfb",66419:"e6a0b17c",66555:"4058ffc7",66808:"061aa888",66945:"1e9742d3",66981:"42ac147f",67218:"4d5c911a",67287:"30019ebd",67394:"0c477410",67448:"ea219a5e",67597:"1f3a4c28",67664:"1f13f78c",67706:"b758af82",67870:"668faecc",68070:"5466f725",68283:"90b5bff2",68314:"5990284f",68526:"93b7d091",68549:"c424522e",68585:"01ca353e",68809:"34affa88",69159:"e2f680f0",69226:"ec923500",69314:"dc5f5244",69559:"fc9487d1",69966:"008e5e0b",70224:"02e885c1",70344:"6ee9dc6c",70406:"5572fd3b",70880:"660ff070",71535:"d1cced29",71580:"60736693",72141:"3c8ed086",72249:"833d6d75",72286:"97cc98df",72293:"5635000a",72326:"24f71dd5",72912:"622d8a7f",73060:"a4a4b44f",73249:"6b863ffe",73419:"78729bc5",73443:"f064c580",73602:"68c9c4d9",73796:"52f85e79",74275:"04909339",74772:"fe2304b2",74963:"3f156112",75088:"4d3332b5",75488:"80bb8867",75586:"ced09747",75658:"39b67445",76258:"e4fa02c3",76300:"de74179d",76454:"392d130c",76515:"09d48160",76572:"844b09b1",76780:"9fcdbad9",76843:"1987905e",77225:"d02aa824",77390:"a83939c0",77491:"28d99640",77682:"dd07edcb",77697:"8ab49adf",77969:"f101a58f",78047:"17c85fed",78480:"6094a26e",78579:"29367017",78817:"861d7487",79043:"9386a8fc",79079:"550ded59",79088:"4dea5ba6",79328:"01e88565",79362:"0185cc0f",80053:"0c4b2f48",80112:"fd699d0f",80322:"c2a5fb58",80598:"9604dd74",80707:"693fd0ce",80751:"17b0c135",80829:"a2ebc88c",80837:"842e3aa3",80874:"bf4d4db4",80982:"3a6c096a",81102:"ddb287b3",81340:"c70c4ed4",81663:"16985740",81787:"e94876f2",81878:"5025bbc2",81913:"45c6b533",81970:"f817585c",82158:"34c7f206",82412:"06ef2967",82569:"b8197956",82643:"aab4adeb",82668:"86174e6f",82681:"431ff015",82723:"2533d7d1",82744:"251a5009",82880:"e515703a",82994:"db964a35",83251:"5b905490",83378:"30d16d45",83393:"b4209ab4",83669:"b09b75c1",83781:"40717c05",83790:"eaac0795",84202:"3d5cb62e",84207:"75958044",84298:"a9f205e4",84880:"ee402f90",84908:"2baaa7ee",84954:"a71c1740",84979:"2a25ce30",85043:"f98b3ebd",85122:"e3efa00f",85332:"4a6b1343",85433:"81df0347",85648:"e46388c0",85729:"0a998fbe",85847:"0064d3f3",86184:"ce1a31c9",86248:"41094952",86482:"79b92687",86704:"4f81ddb2",86856:"985584fd",87320:"e9b88331",87384:"6d8d8c86",87452:"b3e364f6",87597:"41108fd6",87684:"e7b3125d",87780:"f43ae5ba",87808:"37b0acfa",88003:"4b4a7a4c",88838:"1a748968",88984:"3ff08287",89220:"e11e9cc5",89262:"6c87737b",89407:"e589fd66",89450:"f36475fd",89461:"2a186faa",89463:"fcc8cf1f",89767:"5f64aaac",89864:"da57dee1",90288:"f854e6db",90544:"ebad36c0",91078:"8a9dd085",91103:"634f6370",91210:"f250c9ac",91214:"9600caec",91387:"50b3c0bf",91391:"6e8550b2",91570:"c5bc4d89",91747:"1df1ce06",91755:"efaa05fe",91824:"d2381236",91871:"264fce60",91895:"7233fcae",91921:"0f73e2cb",92018:"18420d4b",92045:"b7082472",92059:"3cb43b94",92321:"a13df7c4",92805:"bc260ba1",92836:"e6ce93fd",92867:"86a6b9e8",92982:"902b0046",93089:"5241fd92",93377:"aa0a6603",94330:"c6afff6b",94353:"a5690af9",94395:"8bb62ead",94508:"a7b3b267",94548:"7223c440",94553:"49648640",94579:"17f5fc3c",94597:"985993f6",94603:"107f7123",94623:"552eed53",94702:"239c52b1",94744:"1273506a",94756:"e8ca3715",94916:"ba81f68e",94994:"e5301c26",95049:"1d1e73a8",95064:"a0f894d7",95521:"64a26515",95584:"9c262440",95602:"279c2c7b",95717:"2a4b9c2d",95768:"92ce7313",95974:"53f9c073",96350:"146afc64",96673:"1c5fb002",96816:"d0dd057b",96922:"36781a06",96960:"05a860df",97030:"47d88247",97065:"6050493a",97165:"c0ed39cb",97297:"2ae26e66",97381:"3bd2df9d",97715:"61cec5c2",97721:"489ca716",97800:"28867a86",97920:"b813a053",97988:"956d5ccf",98050:"0cdbfd5b",98197:"69b26a94",98462:"9f99c59e",98528:"f7d9cf13",98546:"238da816",98583:"1cb4799b",99119:"8e792955",99128:"abb2247b",99147:"2e0f9fd2",99161:"3e7e608f",99374:"4be47182",99378:"e5729889",99601:"3a075b02",99781:"0dcdc1ac",99970:"5f073138"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,b)=>Object.prototype.hasOwnProperty.call(e,b),d={},a="synapseml:",r.l=(e,b,f,c)=>{if(d[e])d[e].push(b);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i<n.length;i++){var u=n[i];if(u.getAttribute("src")==e||u.getAttribute("data-webpack")==a+f){t=u;break}}t||(o=!0,(t=document.createElement("script")).charset="utf-8",t.timeout=120,r.nc&&t.setAttribute("nonce",r.nc),t.setAttribute("data-webpack",a+f),t.src=e),d[e]=[b];var l=(b,f)=>{t.onerror=t.onload=null,clearTimeout(s);var a=d[e];if(delete d[e],t.parentNode&&t.parentNode.removeChild(t),a&&a.forEach((e=>e(f))),b)return b(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/SynapseML/",r.gca=function(e){return e={15516451:"99128",17896441:"27918",20641120:"787",26631026:"78480",55203621:"3554",71042800:"59484",75406112:"73060",75801678:"6289",90742975:"36005","32fe2e34":"226",e8d66fc1:"271","88b63415":"360","001aea4c":"390","7b60d8e3":"403",c4d68582:"429",b83021be:"587",b66ab6b5:"688",b3272230:"806","73c2022c":"847","0a65b9de":"939","421d486c":"1084","1f6f5ac9":"1123","255635fc":"1172",f264933a:"1283",fd4bd09d:"1286","3cbfbf75":"1773","513002de":"1802",ad10988d:"1854","368de161":"1985","6c82e982":"2070",b72abe57:"2195","59e4d88e":"2381","09bc1516":"2408",e4e95844:"2797","980b00ea":"2901","6c61f93b":"3202","6e159789":"3611","513214d5":"3701",d1428557:"3765",b41f295c:"3814","445012ce":"3871","60a2189a":"3953","73cf52e8":"4026","1ad91055":"4193",ea2d2bcc:"4204","1e0d2d3b":"4503",c065fcb9:"4800",cf57716c:"5000","405fabd0":"5253","33b3776b":"5254",b8963dc0:"5287",fc338b02:"5378","1155dc6f":"5789","78cb5959":"5909","10b7acf6":"6004","5e008bd1":"6304","4bbbdfcf":"6413","707d2a35":"6633",ed40d5f1:"6712","567cd2fd":"7201","268b7b40":"7234","9530a2cf":"7509","9c5088a2":"7925","9da9112f":"8006",e24ee294:"8040",d13e532e:"8146",e7bd83ca:"8168","2d527871":"8749",a563fa93:"8839","41f327fc":"8966","1518f45a":"9296","4f1ca6a9":"9540","014ce9a3":"10643",e12b4691:"10912",ac581902:"10978",c76e8cac:"11298",fba72e87:"11542","2ece0fbe":"11561","3b5de274":"11977",d05981ec:"12124","6c7ccb6d":"12398","5962ef6e":"12630","75514f6a":"12824","4bea8531":"12840","1b3ab3b8":"13431",ef6ec597:"13482","1b7af47d":"14273",a5ffe972:"14341","1842239f":"14535","90e4432b":"15095","603706f6":"15149","4f81391a":"15165",b7787e72:"15253","1abf1e18":"15343",ac39f7dc:"15404","3a4f9d93":"15487",a9330763:"15511",ebf648dd:"15566",dd29718b:"15579",f3e986ee:"15788","18c7d2a3":"15800","07f6b90a":"15839","0f113696":"16030","76d23901":"16269",bef0d2d3:"16286","3c1f4383":"16551","5b779334":"16833","51dbb02f":"16975","3ea19ec0":"17026","6eddee4c":"17295",ad139e3c:"17538","66d63bfc":"17542",b1706689:"17655",b328a361:"17782","1766722a":"17817",dd3f9ada:"17820","63cd8b22":"17973","445a1b28":"18373",ceeda143:"18443",d84904f9:"18563",c73cfc52:"18825","514f485e":"18828","11c317cb":"18933",c14fdd92:"19042","8bff2444":"19336","548964ce":"19465","15ea8379":"19483","2b30bc14":"19603",b2be3901:"19775","275c0efd":"19950","20e0fe38":"21102","6e4e5cd9":"21401",cb7c2a83:"21933",a34c49d0:"22007",e316de39:"22011","5407376f":"22287",aa5548ab:"22341","8af72580":"22577","49e6864b":"22707","6b40ec54":"22712","88f8e3e1":"22970","052105dd":"23036",c0391845:"23234","4e3910af":"23544",f21c13c4:"23807","92ce4fd5":"23838","35826f14":"23894","2bcda99e":"23941","48329b94":"23979","7aeb6f16":"24046",c5c95e45:"24422","1a3608a9":"24484",b962e007:"24594",e579bad6:"24627","10e13725":"25116",cf666d49:"25449",cee4c3b9:"25844","38d6824e":"25990","295a8e78":"26021","0eaf8468":"26184","03d0d544":"26676",c2e57a1b:"26685","8a1bca9d":"26871",ce40236b:"27497",a07d09c1:"27572","35bdc661":"27813",dc1e40d7:"27991","8d990105":"28456","6e2678b7":"28465",dac82c00:"28559",f1b2dc7f:"28634",b066233f:"28696",d620a661:"28746","6e71bda3":"28955","2d5b0e6a":"28957","9783ff24":"29006","7c007ec6":"29283","1be78505":"29514","905ce8b5":"29535",c82134d8:"29540","166d99f5":"29696",b9f125cc:"29806","39a50b22":"29845","5eddefdd":"30013",b739c60a:"30023","627aefca":"30301","22444eb9":"30335","0a9c6347":"30460","47bbce76":"30464",a6191053:"30927",c95882d3:"31153",fd3b6ea5:"31279","521f9727":"31333","645abc7d":"31836","9fa7e8eb":"31896","561bd03d":"32185","5f17911b":"32599","8ff6772d":"32777","0606f1a5":"32789","5bf420e9":"32855",da508cef:"33064","6a26e359":"33195","5c18deb5":"33358","90f00051":"33631","01587ade":"33988","141dc4a8":"34334","3e1e84da":"34397",e887b273:"34654","0d432c84":"34797","7275f11a":"34940","2c5f0abd":"35015","0daa5b3f":"35024","767a7177":"35040",c3b1d949:"35056","061b38c5":"35124","54287b47":"35328","22f921cc":"35517",c3dac62d:"35537","0637d1e7":"35750","0453ae68":"35939","50af03e4":"36204","6c7d288d":"36224",b0e28723:"36285","589adaf2":"36572",b7226801:"36585",affaa541:"36673",aba932dd:"36698","5ef28062":"36746",b4ae214b:"36773",df01388f:"36828",f027bb15:"36916","1e57c92a":"36958","83586bff":"37001","40b49758":"37069","27de0303":"37334",ef77ef39:"37392","21f47424":"37474",a406e231:"37486","0713a501":"37592","077ca2ec":"37610",d57515f7:"37638","4631a032":"37676","5efd5006":"37742",d9e5e528:"38057","6f8cd013":"38078","012c1c0f":"38261","70b306ba":"38377","327be84b":"38399","16d05917":"38759","5e7c3303":"38781",bf04857e:"38846",b7802ae1:"38859","4b18e5ae":"38962",ae89cd3b:"39139",bb7a48da:"39242","0d6d64cd":"39307","16d8ca91":"39402","4e105cba":"39596","2e93ae0f":"40023",fece3c24:"40367","9e5b788f":"40412","610d430a":"40659","965ed185":"40852","1db1f785":"40950","04804d02":"41607",b66e2100:"41719","391cb159":"41867","79114dfe":"41944","2cec6bcf":"41987","328f3af4":"42037",e3089280:"42283","1789daeb":"42377","31ee0fa3":"42428","6c4c8509":"42522",e21a3367:"42638","3ee83f58":"42802","993c5022":"42815","423540f2":"42852","42736d5f":"42947",de3bb7c3:"42970","1587ba1d":"43156","455aa1bd":"43251","995576e9":"43344","494501bd":"43426","336404dc":"43488","034e6169":"43730",d41234a3:"44177","70fe7dd7":"44178","00ab2cda":"44334",cd8e97a5:"44422",d81d1137:"44490","99e5ffa1":"44504","2d8a44d7":"44534",fe6131e8:"44539","104b9b56":"45282","7649ec47":"45370",bc85b804:"45503","7ad384dc":"45589","8799ef55":"45938",ccc49370:"46103","96cf5ff0":"46135","0fb98d33":"46338","98258c5e":"46559",b4898d44:"46613",c39e3602:"46807","79d79762":"46845",c5775233:"47454","01eb996b":"47552","408524d4":"48007","59ef8022":"48491","25fb872c":"48602","13246fd5":"48807",eed630e0:"49101",efbc69e1:"49196","3e7c5569":"49778","3a7a88af":"49814",fc598b5d:"50002",a81cf8b0:"50040","9c279ae7":"50196",fe2c893b:"50297","97592aac":"50653","069e3c4c":"50745","5c0b440a":"50783",ff428354:"51078","6b9bdd6f":"51105","6a9ad78d":"51317",b4a95996:"51478",e9aab890:"51732",d8d6ba90:"51842","644ff257":"51910","9688a5be":"51918",c4d09a44:"52406","345903d4":"52433","2e5f4263":"52447","814f3328":"52535",a3fd5a6d:"52730","9b020c4a":"52752",ed5c6c48:"52787","12b89d4a":"52804",bf002efe:"53017","943afd32":"53050","86ed1ff2":"53090",b88f1f9b:"53338",a8456d9f:"53572","9e4087bc":"53608",cb5f1a06:"54528",f537d1e9:"54890","76b137bf":"55276",ca0a1bad:"55305",bece3771:"55538","41ee0cff":"55702",ce7af6ae:"55864",cff2e56e:"56373","1293b2b0":"57074",d9837698:"57168","9d095dba":"57231",e2689dfc:"57258","39122aab":"57689",be95fd8f:"57887","15a59c33":"57924",d39aa6d3:"57930","4d31bfb3":"58092",d989c707:"58462","0349e4f0":"58524",b940e0b5:"58738","990f07de":"58755","3bed3e2f":"58846","7c2e1100":"58861",dfb6dfe9:"59076","4801d370":"59394","76bf568a":"59823","403e642d":"60055",f322d2da:"60445",f0649d54:"60880","3ef20ed6":"61407",b5375b6f:"61428",b75118f0:"61442","04fc65cc":"61755",a0d8a1a2:"61778",f38a0cbd:"61829","5cfa133c":"62079","2c98ca3e":"62106","04ee4ab4":"62217",ddacd194:"62465",c38c658c:"62565","4ef2f7cf":"62675",d95a70ea:"62721","52880d18":"62750","08019edc":"62773",bf8a5c8a:"63002","95d9e891":"63099","74091a62":"63196","56543a15":"63402",ed6d544d:"63448","3bbe0f34":"64172",c4f5d8e4:"64195",b7c6295c:"64233","0e3f998d":"64482",d355047f:"64713",fa2e2688:"64927","37ab9beb":"64992","44ff0d07":"65437",c3c516ff:"65752","900b935c":"65942","74659d33":"66167",e5fe80e3:"66419","0fa36e32":"66555","8462374b":"66808","296054a7":"66945",e863531f:"66981",ba9c0924:"67218","99f8fee5":"67287","31bb7274":"67394","888e2229":"67448",a608660b:"67597",faca745a:"67664",db170a23:"67706",ed316aed:"67870",a1824f78:"68070",e84755e1:"68283","104bf0a6":"68314","3201258e":"68526",c991f47b:"68549","18c9c2ba":"68585",fd748117:"68809","57e687e8":"69159","8181e18a":"69226","232d09b0":"69314","3fb29942":"69559",d1246f49:"69966","77c259dd":"70224","7aa7b8e0":"70344","6a07b55e":"70406","2461af4d":"70880","1a215693":"71535","179d57fa":"71580",a877f9e5:"72141","38b375cc":"72249","3d963cd5":"72286","56087ddb":"72293","1ef65056":"72326","13f17286":"72912","5225b7e0":"73249","8e32e44d":"73419","91e9cb67":"73443",af1e70b6:"73602","6a95f87e":"73796","792eae7e":"74275","4997ef4f":"74772","6f479459":"74963","7a1d9beb":"75088",cbc3190d:"75488",e7250a93:"75586",c23b5e26:"75658","3dafa002":"76258",d252041b:"76300","34f00221":"76454","9cb6fa7c":"76515","58486a98":"76572","62983deb":"76843","51878b77":"77225",b35e9998:"77390",d3093636:"77491","50f80512":"77682","39582d99":"77697","5887dc62":"77969","3b75a1c6":"78047","0b416bde":"78579","88e44e99":"78817","00353f68":"79043",efc1978d:"79079","0a954229":"79088",a0434473:"79328","34ff21b6":"79362","935f2afb":"80053","8fd0c721":"80112",b576c645:"80322",e1269457:"80598",ab9a8589:"80707","556ad472":"80751",ee6cea3b:"80829","17f418b7":"80837",b92ffdc6:"80874","2fb6c1b0":"80982",d25cde70:"81102",e1b2b5f3:"81340","00056db7":"81663",e4ba4487:"81787",fa889bee:"81878","1d820955":"81913","01a15f20":"81970",f7580424:"82158","12d85ab5":"82412",e03f5a54:"82569",f228fdc5:"82643","3a1147b5":"82668","62ccfbde":"82681","23af8e3e":"82723",f8597cb0:"82744",e1ed1e14:"82880","48cf7354":"82994",c34b1e36:"83251",fbb011bb:"83378","3e9a7422":"83393",d707d2ed:"83669","866b91a1":"83781","604b2d50":"83790",c2492d56:"84202",f14633e7:"84207","6094981e":"84298",f8aab3ca:"84880","59d9fecc":"84908",f7a32432:"84954",e8b9c2c7:"84979","8eb438b4":"85043",c2e40c34:"85122","321ba9c5":"85332",ffb7f88a:"85433",ee83122d:"85648","56d039a1":"85729","4e044b4e":"85847","4130db01":"86184","7bf16cc6":"86248","3c22aa66":"86482","67e06a8b":"86704",b80b493a:"86856",c40984d1:"87320",b35d004b:"87384","93ba15e0":"87452","66fcb786":"87597",b38eec89:"87684",b28fe732:"87808","5066efb2":"88003","70e8008e":"88838",d535b358:"88984",a2b5e5df:"89220","120352d6":"89262",deef0af8:"89407","83203dd1":"89450","6ff7775f":"89461",a19ce767:"89463",c413b43a:"89767","1dc2f362":"89864","69bb4b11":"90288","4585b70e":"90544","80472ecc":"91078","3ac37a06":"91103",e2ce57ed:"91210",cc52e3cd:"91214",d36f3429:"91387","2137a7cd":"91391","4de42975":"91570","85aed2f9":"91747",bc4bf151:"91755",c90c942b:"91824","8590feaa":"91871","78aebd5d":"91895","8328d740":"91921",dc7f7ac1:"92018","0e0ee9b2":"92045",b33598ac:"92059","325e22bd":"92321",d175df5e:"92805","8a46fce0":"92836","5dbf4619":"92867","3647ac9b":"92982",a6aa9e1f:"93089","318ccb43":"93377",d8b3ac6a:"94330","1f31a275":"94353","0e2bf69e":"94395",e464d112:"94508","53a5cb1e":"94548",d7f58694:"94553","94d743d6":"94579",d0cb74df:"94597",f7bfff73:"94603",e402e26d:"94623","5a3e1bca":"94702","1014102e":"94744","4cee39d7":"94756","82878c84":"94916","109f6864":"94994",f85db098:"95049",e9b75d67:"95064","283b8f99":"95521",bd1bcb2c:"95584","57ef17c5":"95602",d0c5e4fa:"95717",a34b09d4:"95768","8cb11643":"95974","1e14124f":"96350",e5c4ef01:"96673","82d7cc79":"96816","00e0a48e":"96922","1a62d1af":"96960",c764f2c0:"97030",fe5d19cb:"97065","294c060e":"97165",e0077969:"97297","4db82253":"97381","2a023663":"97715","0e16e9ea":"97721","09990b79":"97800","1a4e3797":"97920","4e432063":"97988",d3c9b0d7:"98050",eb0ef952:"98197",bc6a9944:"98462","7296bed0":"98528",f5b4e517:"98546","6bdbf6b8":"98583","2406587d":"99119","7aa49360":"99147","69dc6d0a":"99161","58b8f176":"99374",efbb59fb:"99378",fc47fc53:"99601",dea122e2:"99781","093b81f4":"99970"}[e]||e,r.p+r.u(e)},(()=>{var e={51303:0,40532:0};r.f.j=(b,f)=>{var d=r.o(e,b)?e[b]:void 0;if(0!==d)if(d)f.push(d[2]);else if(/^(40532|51303)$/.test(b))e[b]=0;else{var a=new Promise(((f,a)=>d=e[b]=[f,a]));f.push(d[2]=a);var c=r.p+r.u(b),t=new Error;r.l(c,(f=>{if(r.o(e,b)&&(0!==(d=e[b])&&(e[b]=void 0),d)){var a=f&&("load"===f.type?"missing":f.type),c=f&&f.target&&f.target.src;t.message="Loading chunk "+b+" failed.\n("+a+": "+c+")",t.name="ChunkLoadError",t.type=a,t.request=c,d[1](t)}}),"chunk-"+b,b)}},r.O.j=b=>0===e[b];var b=(b,f)=>{var d,a,c=f[0],t=f[1],o=f[2],n=0;if(c.some((b=>0!==e[b]))){for(d in t)r.o(t,d)&&(r.m[d]=t[d]);if(o)var i=o(r)}for(b&&b(f);n<c.length;n++)a=c[n],r.o(e,a)&&e[a]&&e[a][0](),e[a]=0;return r.O(i)},f=self.webpackChunksynapseml=self.webpackChunksynapseml||[];f.forEach(b.bind(null,0)),f.push=b.bind(null,f.push.bind(f))})()})();
\ No newline at end of file
diff --git a/assets/js/runtime~main.ea772462.js b/assets/js/runtime~main.ea772462.js
deleted file mode 100644
index 41212e38b3..0000000000
--- a/assets/js/runtime~main.ea772462.js
+++ /dev/null
@@ -1 +0,0 @@
-(()=>{"use strict";var e,b,f,d,a,c={},t={};function r(e){var b=t[e];if(void 0!==b)return b.exports;var f=t[e]={exports:{}};return c[e].call(f.exports,f,f.exports,r),f.exports}r.m=c,e=[],r.O=(b,f,d,a)=>{if(!f){var c=1/0;for(i=0;i<e.length;i++){f=e[i][0],d=e[i][1],a=e[i][2];for(var t=!0,o=0;o<f.length;o++)(!1&a||c>=a)&&Object.keys(r.O).every((e=>r.O[e](f[o])))?f.splice(o--,1):(t=!1,a<c&&(c=a));if(t){e.splice(i--,1);var n=d();void 0!==n&&(b=n)}}return b}a=a||0;for(var i=e.length;i>0&&e[i-1][2]>a;i--)e[i]=e[i-1];e[i]=[f,d,a]},r.n=e=>{var b=e&&e.__esModule?()=>e.default:()=>e;return r.d(b,{a:b}),b},f=Object.getPrototypeOf?e=>Object.getPrototypeOf(e):e=>e.__proto__,r.t=function(e,d){if(1&d&&(e=this(e)),8&d)return e;if("object"==typeof e&&e){if(4&d&&e.__esModule)return e;if(16&d&&"function"==typeof e.then)return e}var a=Object.create(null);r.r(a);var c={};b=b||[null,f({}),f([]),f(f)];for(var t=2&d&&e;"object"==typeof t&&!~b.indexOf(t);t=f(t))Object.getOwnPropertyNames(t).forEach((b=>c[b]=()=>e[b]));return c.default=()=>e,r.d(a,c),a},r.d=(e,b)=>{for(var f in b)r.o(b,f)&&!r.o(e,f)&&Object.defineProperty(e,f,{enumerable:!0,get:b[f]})},r.f={},r.e=e=>Promise.all(Object.keys(r.f).reduce(((b,f)=>(r.f[f](e,b),b)),[])),r.u=e=>"assets/js/"+({226:"32fe2e34",271:"e8d66fc1",360:"88b63415",390:"001aea4c",403:"7b60d8e3",429:"c4d68582",587:"b83021be",688:"b66ab6b5",787:"20641120",806:"b3272230",847:"73c2022c",939:"0a65b9de",1084:"421d486c",1123:"1f6f5ac9",1172:"255635fc",1283:"f264933a",1286:"fd4bd09d",1773:"3cbfbf75",1802:"513002de",1854:"ad10988d",1985:"368de161",2070:"6c82e982",2195:"b72abe57",2381:"59e4d88e",2408:"09bc1516",2797:"e4e95844",2901:"980b00ea",3202:"6c61f93b",3554:"55203621",3611:"6e159789",3701:"513214d5",3765:"d1428557",3814:"b41f295c",3871:"445012ce",3953:"60a2189a",4026:"73cf52e8",4193:"1ad91055",4204:"ea2d2bcc",4503:"1e0d2d3b",4800:"c065fcb9",5e3:"cf57716c",5253:"405fabd0",5254:"33b3776b",5287:"b8963dc0",5378:"fc338b02",5789:"1155dc6f",5909:"78cb5959",6004:"10b7acf6",6289:"75801678",6304:"5e008bd1",6413:"4bbbdfcf",6633:"707d2a35",6712:"ed40d5f1",7201:"567cd2fd",7234:"268b7b40",7509:"9530a2cf",7925:"9c5088a2",8006:"9da9112f",8040:"e24ee294",8146:"d13e532e",8168:"e7bd83ca",8749:"2d527871",8839:"a563fa93",8966:"41f327fc",9296:"1518f45a",9540:"4f1ca6a9",10643:"014ce9a3",10912:"e12b4691",10978:"ac581902",11298:"c76e8cac",11542:"fba72e87",11561:"2ece0fbe",11977:"3b5de274",12124:"d05981ec",12398:"6c7ccb6d",12630:"5962ef6e",12824:"75514f6a",12840:"4bea8531",13431:"1b3ab3b8",13482:"ef6ec597",14273:"1b7af47d",14341:"a5ffe972",14535:"1842239f",15095:"90e4432b",15149:"603706f6",15165:"4f81391a",15253:"b7787e72",15343:"1abf1e18",15404:"ac39f7dc",15487:"3a4f9d93",15511:"a9330763",15566:"ebf648dd",15579:"dd29718b",15788:"f3e986ee",15800:"18c7d2a3",15839:"07f6b90a",16030:"0f113696",16269:"76d23901",16286:"bef0d2d3",16551:"3c1f4383",16833:"5b779334",16975:"51dbb02f",17026:"3ea19ec0",17295:"6eddee4c",17538:"ad139e3c",17542:"66d63bfc",17655:"b1706689",17782:"b328a361",17817:"1766722a",17820:"dd3f9ada",17973:"63cd8b22",18373:"445a1b28",18443:"ceeda143",18563:"d84904f9",18825:"c73cfc52",18828:"514f485e",18933:"11c317cb",19042:"c14fdd92",19336:"8bff2444",19465:"548964ce",19483:"15ea8379",19603:"2b30bc14",19775:"b2be3901",19950:"275c0efd",21102:"20e0fe38",21401:"6e4e5cd9",21933:"cb7c2a83",22007:"a34c49d0",22011:"e316de39",22287:"5407376f",22341:"aa5548ab",22577:"8af72580",22707:"49e6864b",22712:"6b40ec54",22970:"88f8e3e1",23036:"052105dd",23234:"c0391845",23544:"4e3910af",23807:"f21c13c4",23838:"92ce4fd5",23894:"35826f14",23941:"2bcda99e",23979:"48329b94",24046:"7aeb6f16",24422:"c5c95e45",24484:"1a3608a9",24594:"b962e007",24627:"e579bad6",25116:"10e13725",25449:"cf666d49",25844:"cee4c3b9",25990:"38d6824e",26021:"295a8e78",26184:"0eaf8468",26676:"03d0d544",26685:"c2e57a1b",26871:"8a1bca9d",27497:"ce40236b",27572:"a07d09c1",27813:"35bdc661",27918:"17896441",27991:"dc1e40d7",28456:"8d990105",28465:"6e2678b7",28559:"dac82c00",28634:"f1b2dc7f",28696:"b066233f",28746:"d620a661",28955:"6e71bda3",28957:"2d5b0e6a",29006:"9783ff24",29283:"7c007ec6",29514:"1be78505",29535:"905ce8b5",29540:"c82134d8",29696:"166d99f5",29806:"b9f125cc",29845:"39a50b22",30013:"5eddefdd",30023:"b739c60a",30301:"627aefca",30335:"22444eb9",30460:"0a9c6347",30464:"47bbce76",30927:"a6191053",31153:"c95882d3",31279:"fd3b6ea5",31333:"521f9727",31836:"645abc7d",31896:"9fa7e8eb",32185:"561bd03d",32599:"5f17911b",32777:"8ff6772d",32789:"0606f1a5",32855:"5bf420e9",33064:"da508cef",33195:"6a26e359",33358:"5c18deb5",33631:"90f00051",33988:"01587ade",34334:"141dc4a8",34397:"3e1e84da",34654:"e887b273",34797:"0d432c84",34940:"7275f11a",35015:"2c5f0abd",35024:"0daa5b3f",35040:"767a7177",35056:"c3b1d949",35124:"061b38c5",35328:"54287b47",35517:"22f921cc",35537:"c3dac62d",35750:"0637d1e7",35939:"0453ae68",36005:"90742975",36204:"50af03e4",36224:"6c7d288d",36285:"b0e28723",36572:"589adaf2",36585:"b7226801",36673:"affaa541",36698:"aba932dd",36746:"5ef28062",36773:"b4ae214b",36828:"df01388f",36916:"f027bb15",36958:"1e57c92a",37001:"83586bff",37069:"40b49758",37334:"27de0303",37392:"ef77ef39",37474:"21f47424",37486:"a406e231",37592:"0713a501",37610:"077ca2ec",37638:"d57515f7",37676:"4631a032",37742:"5efd5006",38057:"d9e5e528",38078:"6f8cd013",38261:"012c1c0f",38377:"70b306ba",38399:"327be84b",38759:"16d05917",38781:"5e7c3303",38846:"bf04857e",38859:"b7802ae1",38962:"4b18e5ae",39139:"ae89cd3b",39242:"bb7a48da",39307:"0d6d64cd",39402:"16d8ca91",39596:"4e105cba",40023:"2e93ae0f",40367:"fece3c24",40412:"9e5b788f",40659:"610d430a",40852:"965ed185",40950:"1db1f785",41607:"04804d02",41719:"b66e2100",41867:"391cb159",41944:"79114dfe",41987:"2cec6bcf",42037:"328f3af4",42283:"e3089280",42377:"1789daeb",42428:"31ee0fa3",42522:"6c4c8509",42638:"e21a3367",42802:"3ee83f58",42815:"993c5022",42852:"423540f2",42947:"42736d5f",42970:"de3bb7c3",43156:"1587ba1d",43251:"455aa1bd",43344:"995576e9",43426:"494501bd",43488:"336404dc",43730:"034e6169",44177:"d41234a3",44178:"70fe7dd7",44334:"00ab2cda",44422:"cd8e97a5",44490:"d81d1137",44504:"99e5ffa1",44534:"2d8a44d7",44539:"fe6131e8",45282:"104b9b56",45370:"7649ec47",45503:"bc85b804",45589:"7ad384dc",45938:"8799ef55",46103:"ccc49370",46135:"96cf5ff0",46338:"0fb98d33",46559:"98258c5e",46613:"b4898d44",46807:"c39e3602",46845:"79d79762",47454:"c5775233",47552:"01eb996b",48007:"408524d4",48491:"59ef8022",48602:"25fb872c",48807:"13246fd5",49101:"eed630e0",49196:"efbc69e1",49778:"3e7c5569",49814:"3a7a88af",50002:"fc598b5d",50040:"a81cf8b0",50196:"9c279ae7",50297:"fe2c893b",50653:"97592aac",50745:"069e3c4c",50783:"5c0b440a",51078:"ff428354",51105:"6b9bdd6f",51317:"6a9ad78d",51478:"b4a95996",51732:"e9aab890",51842:"d8d6ba90",51910:"644ff257",51918:"9688a5be",52406:"c4d09a44",52433:"345903d4",52447:"2e5f4263",52535:"814f3328",52730:"a3fd5a6d",52752:"9b020c4a",52787:"ed5c6c48",52804:"12b89d4a",53017:"bf002efe",53050:"943afd32",53090:"86ed1ff2",53338:"b88f1f9b",53572:"a8456d9f",53608:"9e4087bc",54528:"cb5f1a06",54890:"f537d1e9",55276:"76b137bf",55305:"ca0a1bad",55538:"bece3771",55702:"41ee0cff",55864:"ce7af6ae",56373:"cff2e56e",57074:"1293b2b0",57168:"d9837698",57231:"9d095dba",57258:"e2689dfc",57689:"39122aab",57887:"be95fd8f",57924:"15a59c33",57930:"d39aa6d3",58092:"4d31bfb3",58462:"d989c707",58524:"0349e4f0",58738:"b940e0b5",58755:"990f07de",58846:"3bed3e2f",58861:"7c2e1100",59076:"dfb6dfe9",59394:"4801d370",59484:"71042800",59823:"76bf568a",60055:"403e642d",60445:"f322d2da",60880:"f0649d54",61407:"3ef20ed6",61428:"b5375b6f",61442:"b75118f0",61755:"04fc65cc",61778:"a0d8a1a2",61829:"f38a0cbd",62079:"5cfa133c",62106:"2c98ca3e",62217:"04ee4ab4",62465:"ddacd194",62565:"c38c658c",62675:"4ef2f7cf",62721:"d95a70ea",62750:"52880d18",62773:"08019edc",63002:"bf8a5c8a",63099:"95d9e891",63196:"74091a62",63402:"56543a15",63448:"ed6d544d",64172:"3bbe0f34",64195:"c4f5d8e4",64233:"b7c6295c",64482:"0e3f998d",64713:"d355047f",64927:"fa2e2688",64992:"37ab9beb",65437:"44ff0d07",65752:"c3c516ff",65942:"900b935c",66167:"74659d33",66419:"e5fe80e3",66555:"0fa36e32",66808:"8462374b",66945:"296054a7",66981:"e863531f",67218:"ba9c0924",67287:"99f8fee5",67394:"31bb7274",67448:"888e2229",67597:"a608660b",67664:"faca745a",67706:"db170a23",67870:"ed316aed",68070:"a1824f78",68283:"e84755e1",68314:"104bf0a6",68526:"3201258e",68549:"c991f47b",68585:"18c9c2ba",68809:"fd748117",69159:"57e687e8",69226:"8181e18a",69314:"232d09b0",69559:"3fb29942",69966:"d1246f49",70224:"77c259dd",70344:"7aa7b8e0",70406:"6a07b55e",70880:"2461af4d",71535:"1a215693",71580:"179d57fa",72141:"a877f9e5",72249:"38b375cc",72286:"3d963cd5",72293:"56087ddb",72326:"1ef65056",72912:"13f17286",73060:"75406112",73249:"5225b7e0",73419:"8e32e44d",73443:"91e9cb67",73602:"af1e70b6",73796:"6a95f87e",74275:"792eae7e",74772:"4997ef4f",74963:"6f479459",75088:"7a1d9beb",75488:"cbc3190d",75586:"e7250a93",75658:"c23b5e26",76258:"3dafa002",76300:"d252041b",76454:"34f00221",76515:"9cb6fa7c",76572:"58486a98",76843:"62983deb",77225:"51878b77",77390:"b35e9998",77491:"d3093636",77682:"50f80512",77697:"39582d99",77969:"5887dc62",78047:"3b75a1c6",78480:"26631026",78579:"0b416bde",78817:"88e44e99",79043:"00353f68",79079:"efc1978d",79088:"0a954229",79328:"a0434473",79362:"34ff21b6",80053:"935f2afb",80112:"8fd0c721",80322:"b576c645",80598:"e1269457",80707:"ab9a8589",80751:"556ad472",80829:"ee6cea3b",80837:"17f418b7",80874:"b92ffdc6",80982:"2fb6c1b0",81102:"d25cde70",81340:"e1b2b5f3",81663:"00056db7",81787:"e4ba4487",81878:"fa889bee",81913:"1d820955",81970:"01a15f20",82158:"f7580424",82412:"12d85ab5",82569:"e03f5a54",82643:"f228fdc5",82668:"3a1147b5",82681:"62ccfbde",82723:"23af8e3e",82744:"f8597cb0",82880:"e1ed1e14",82994:"48cf7354",83251:"c34b1e36",83378:"fbb011bb",83393:"3e9a7422",83669:"d707d2ed",83781:"866b91a1",83790:"604b2d50",84202:"c2492d56",84207:"f14633e7",84298:"6094981e",84880:"f8aab3ca",84908:"59d9fecc",84954:"f7a32432",84979:"e8b9c2c7",85043:"8eb438b4",85122:"c2e40c34",85332:"321ba9c5",85433:"ffb7f88a",85648:"ee83122d",85729:"56d039a1",85847:"4e044b4e",86184:"4130db01",86248:"7bf16cc6",86482:"3c22aa66",86704:"67e06a8b",86856:"b80b493a",87320:"c40984d1",87384:"b35d004b",87452:"93ba15e0",87597:"66fcb786",87684:"b38eec89",87808:"b28fe732",88003:"5066efb2",88838:"70e8008e",88984:"d535b358",89220:"a2b5e5df",89262:"120352d6",89407:"deef0af8",89450:"83203dd1",89461:"6ff7775f",89463:"a19ce767",89767:"c413b43a",89864:"1dc2f362",90288:"69bb4b11",90544:"4585b70e",91078:"80472ecc",91103:"3ac37a06",91210:"e2ce57ed",91214:"cc52e3cd",91387:"d36f3429",91391:"2137a7cd",91570:"4de42975",91747:"85aed2f9",91755:"bc4bf151",91824:"c90c942b",91871:"8590feaa",91895:"78aebd5d",91921:"8328d740",92018:"dc7f7ac1",92045:"0e0ee9b2",92059:"b33598ac",92321:"325e22bd",92805:"d175df5e",92836:"8a46fce0",92867:"5dbf4619",92982:"3647ac9b",93089:"a6aa9e1f",93377:"318ccb43",94330:"d8b3ac6a",94353:"1f31a275",94395:"0e2bf69e",94508:"e464d112",94548:"53a5cb1e",94553:"d7f58694",94579:"94d743d6",94597:"d0cb74df",94603:"f7bfff73",94623:"e402e26d",94702:"5a3e1bca",94744:"1014102e",94756:"4cee39d7",94916:"82878c84",94994:"109f6864",95049:"f85db098",95064:"e9b75d67",95521:"283b8f99",95584:"bd1bcb2c",95602:"57ef17c5",95717:"d0c5e4fa",95768:"a34b09d4",95974:"8cb11643",96350:"1e14124f",96673:"e5c4ef01",96816:"82d7cc79",96960:"1a62d1af",97030:"c764f2c0",97065:"fe5d19cb",97165:"294c060e",97297:"e0077969",97381:"4db82253",97715:"2a023663",97721:"0e16e9ea",97800:"09990b79",97920:"1a4e3797",97988:"4e432063",98050:"d3c9b0d7",98197:"eb0ef952",98462:"bc6a9944",98528:"7296bed0",98546:"f5b4e517",98583:"6bdbf6b8",99119:"2406587d",99128:"15516451",99147:"7aa49360",99161:"69dc6d0a",99374:"58b8f176",99378:"efbb59fb",99601:"fc47fc53",99781:"dea122e2",99970:"093b81f4"}[e]||e)+"."+{226:"7c60e348",271:"b472ae37",360:"0c049a9a",390:"16907556",403:"4c6e2b65",429:"c530f0a0",587:"25e7c11f",688:"feefe5ad",787:"4d89b7a8",806:"3689025c",847:"c07433f0",939:"dd4545ed",1084:"0194791b",1123:"640e13d4",1172:"105f1ab1",1283:"7a0aa43b",1286:"6ad5a370",1773:"3a54b67d",1802:"613317d6",1854:"c62a83e1",1985:"a23ad2e8",2070:"1b8bc6df",2195:"a42749a0",2381:"a9df8fd6",2408:"f68bf61a",2797:"ec2ce8dc",2901:"f0462d81",3202:"83225189",3554:"c9c5c25c",3611:"84cee5cb",3701:"ebe9d132",3765:"c7b94ddd",3814:"a51776da",3871:"3d351853",3953:"aa3ad659",4026:"91ffb89c",4193:"0c39c6fd",4204:"2942d62d",4503:"7ec496a3",4800:"f4fea7cb",5e3:"c4a07252",5253:"58f5ca42",5254:"d9a92894",5287:"b75edc26",5378:"be4cc3ff",5789:"294730b0",5909:"2b553671",6004:"fbfda01a",6289:"7e9ad05f",6304:"e1bb2c7e",6413:"406cd688",6633:"eeed6b75",6712:"11e439bd",7201:"97d9fe39",7234:"61faaca9",7509:"e0320a5b",7925:"45be218c",8006:"696da333",8040:"24249473",8146:"e8787811",8168:"805ffacc",8749:"fd6620c4",8839:"ef3c8cef",8966:"d111deb6",9296:"415afda5",9540:"fd3585d8",10643:"9ac2966d",10912:"5766341c",10978:"fa5141d9",11298:"a8bed199",11542:"4e543c37",11561:"4f426cb6",11977:"4f629e6e",12124:"92c4c2bb",12398:"ab17108f",12630:"89d0fd21",12824:"deab964d",12840:"15a46ad4",13431:"5b3464d7",13482:"f117cd47",14273:"ed1b0a2e",14341:"5f31f2f7",14535:"e5b29187",15095:"1fe4ec82",15149:"b1d50f20",15165:"9fc5aa61",15253:"eb95bb49",15343:"b4f9c1d7",15404:"d4e54d08",15487:"e356abfe",15511:"8de44ef4",15566:"7af2a42d",15579:"80d18a93",15788:"aaa06e2d",15800:"0daa1137",15839:"f15dc558",16030:"a1d2c595",16269:"2a5ae6a5",16286:"3091a1cb",16551:"8334c80a",16833:"ceeed8df",16975:"25f84d8d",17026:"40a33067",17295:"e7002340",17538:"68b2fa1b",17542:"b345b319",17655:"caf41e88",17782:"9af38353",17817:"7a60488d",17820:"7a974bf8",17973:"49a55e91",18373:"3b52ab84",18443:"8ed83508",18563:"a650d3a8",18825:"edd2110a",18828:"61dff53a",18894:"8ba79e33",18933:"82ff982d",19042:"f0e3a658",19336:"7f5c5c75",19465:"ba302a23",19483:"a04abc04",19603:"63140d8c",19775:"53d720b7",19950:"8e6a0524",21102:"e64026a7",21401:"25b8730b",21933:"284bd8a5",22007:"3c7d5195",22011:"12fd87b1",22287:"c1ff4071",22341:"a7c000a6",22577:"12ecb1e2",22707:"969377ed",22712:"204c8f47",22970:"02f3aa52",23036:"ba928913",23234:"6de1ab83",23544:"a2c9c406",23807:"f8ab332b",23838:"f96fd2ce",23894:"4a36373d",23941:"7eb4157a",23979:"da7d92d4",24046:"c95f1e59",24422:"14588453",24484:"290578f8",24572:"7376a9ae",24594:"aa995eb4",24627:"ad35e092",25116:"cadea3bd",25449:"28595d1a",25844:"67b2a5fc",25990:"d1dad3f6",26021:"6b9ce1ea",26184:"d1fb0dee",26676:"e6e7e7a4",26685:"04f76c9c",26871:"25fabaa5",27497:"8be44fac",27572:"92e50813",27813:"7670b946",27918:"1dd7c1ea",27991:"5af43466",28456:"42e1f8bb",28465:"bae03a70",28559:"53d784db",28634:"07a76afe",28696:"03d2fb68",28746:"258f03a6",28955:"3d9432bb",28957:"5c5fc2a0",29006:"bcf81dc8",29283:"a12f8681",29514:"216cf5d7",29535:"75ea1aee",29540:"3ae661ae",29696:"09a48be2",29806:"e13efecc",29845:"6229c1ae",30013:"9156458e",30023:"f70eac19",30301:"c4bbb92d",30335:"ebddc206",30460:"c79f944d",30464:"cfaedc94",30927:"5f7ddb44",31153:"c74b2845",31279:"771e833b",31333:"714da6d3",31836:"6afbe490",31896:"cce4e3d2",32185:"242c68ad",32599:"6ee262cb",32777:"9a4fbe82",32789:"d5b89e35",32855:"ab34218c",33064:"2b15599a",33195:"23458ac7",33358:"6875ed7c",33631:"a5a75b4b",33988:"0caae640",34334:"3a1990ef",34397:"50e63b4b",34654:"96dfebcb",34797:"cf49ef0b",34940:"7bb97a11",35015:"90bcb885",35024:"77f9b663",35040:"ba559dd9",35056:"6d5d38eb",35124:"6cc60e98",35328:"2329ba8e",35517:"5666354f",35537:"27d90dc8",35750:"064379cf",35939:"4fca9411",36005:"07beb68e",36204:"1b5a72e3",36224:"169580d7",36285:"770abe5c",36572:"3b4a629e",36585:"1717b0cf",36673:"6015f67e",36698:"24aeb102",36746:"b6d58204",36773:"a3eb6850",36828:"8ab9b559",36916:"627e234a",36958:"2b54b1f9",37001:"d663eac3",37069:"a5610a85",37334:"bd502a3e",37392:"3674eb36",37474:"8fc13a5a",37486:"be80e33d",37592:"84e46938",37610:"5ca2e1ca",37638:"6fbb6657",37676:"98d2551a",37742:"a5e70451",37853:"6b0071e3",38057:"0aa70e90",38078:"eaa3b4f7",38261:"3e7aa5d5",38377:"bd09de84",38399:"4a04eae7",38759:"873ccf3d",38781:"2f3351a0",38846:"88bd3c18",38859:"7ccc674c",38962:"0236cee6",39139:"6f56c354",39242:"40e4acd1",39307:"5d7232c1",39402:"d6599b0a",39596:"511860eb",40023:"caff10d8",40367:"8325e1db",40412:"8ecdd605",40659:"c8eb504a",40852:"b29206c4",40950:"7be04e11",41607:"38ca02bf",41719:"53c08f2d",41867:"7db460cb",41944:"4ec7bb98",41987:"0609e7ff",42037:"ad3a366f",42283:"a34f8074",42377:"4d69dc8d",42428:"7e0359d6",42522:"096248d0",42638:"082230ce",42802:"91ab538d",42815:"971a7a1a",42852:"bd015a5e",42947:"173cf0a5",42970:"0ca4866d",43156:"6e858d37",43251:"002f23aa",43344:"a5f5a399",43426:"97ac4633",43488:"3bcd1836",43730:"a22706a2",44177:"a7c47f47",44178:"6168e9e1",44334:"7f2d19ca",44422:"e1c7affe",44490:"901cceab",44504:"b8b106d8",44534:"7f1b5fc9",44539:"558a2d88",45282:"1eae2722",45370:"0d930dbf",45503:"a72ce2a2",45589:"7f42a2b5",45938:"ee06af10",46048:"f55105b0",46103:"4043289d",46135:"a123dbdf",46338:"073f7a96",46559:"d0e09414",46613:"0384012a",46807:"62a2c61f",46845:"4deeece5",46945:"fd128107",47454:"2e3cc69a",47552:"96f02664",48007:"1f47865f",48491:"2a20a978",48602:"f679dc4f",48807:"d5a3f483",49101:"a98291dd",49196:"867a0f29",49778:"cd04be09",49814:"cbb3c3ac",50002:"0471b2c6",50040:"e195e324",50196:"cf79206b",50297:"4da5326f",50653:"288194f1",50745:"fc78265d",50783:"3c16a52b",51078:"23039dab",51105:"9273b6ea",51317:"ff17fe39",51478:"f6b57981",51732:"ea31977e",51842:"a88f573c",51910:"eff31342",51918:"0614f4d5",52406:"4d053caf",52433:"30590104",52447:"d06ddc63",52535:"b03595a3",52730:"7a6b4a14",52752:"355782e7",52787:"838e0402",52804:"eefbdfaa",53017:"9348a25f",53050:"6abd1a54",53090:"75338eaf",53140:"a29368cb",53338:"083cbf6d",53572:"7effd19a",53608:"72da96df",54528:"ff5b94ed",54890:"af1bc49c",55276:"facd650b",55305:"1e659f3b",55538:"81957284",55702:"24761a8e",55864:"16c1759c",56373:"f0ab3974",57074:"b14d0343",57168:"9a260310",57231:"f08ac08d",57258:"87fca17c",57689:"0759cd84",57887:"3c00aa5f",57924:"929e4f47",57930:"1e91e4cb",58092:"80d71d4e",58462:"ef15d1f1",58524:"b857b4f3",58738:"43f88bbc",58755:"70e7bcad",58846:"bbcec631",58861:"99c57d57",59076:"823e2f17",59394:"989a950f",59484:"edba3628",59823:"c4a7067d",60055:"fb65aa03",60445:"deb5d152",60880:"4828013e",61407:"8dcb13c6",61428:"d1a4d228",61442:"6214188f",61755:"85679afc",61778:"42efbba1",61829:"79c4438f",62079:"3d50e52d",62106:"b11711e1",62217:"f597b4a5",62465:"26755e65",62565:"7331c10f",62675:"0c1bfb62",62721:"a55c8bbb",62750:"e185638d",62773:"35944844",63002:"cce5fea8",63099:"11b155f6",63196:"3a5e22f4",63402:"9a4cd328",63448:"6d9cca38",64172:"85cd1357",64195:"13fd239b",64233:"871b4ef9",64482:"d6d86e59",64713:"ce848f45",64927:"082fe1c7",64992:"2852555a",65437:"08ba8e6c",65752:"d9a972a6",65942:"3ac80a08",66167:"22cd1bfb",66419:"e6a0b17c",66555:"4058ffc7",66808:"061aa888",66945:"1e9742d3",66981:"42ac147f",67218:"4d5c911a",67287:"30019ebd",67394:"0c477410",67448:"ea219a5e",67597:"1f3a4c28",67664:"1f13f78c",67706:"b758af82",67870:"668faecc",68070:"5466f725",68283:"90b5bff2",68314:"5990284f",68526:"93b7d091",68549:"c424522e",68585:"01ca353e",68809:"34affa88",69159:"e2f680f0",69226:"ec923500",69314:"dc5f5244",69559:"fc9487d1",69966:"008e5e0b",70224:"02e885c1",70344:"6ee9dc6c",70406:"5572fd3b",70880:"660ff070",71535:"d1cced29",71580:"60736693",72141:"3c8ed086",72249:"833d6d75",72286:"97cc98df",72293:"5635000a",72326:"24f71dd5",72912:"622d8a7f",73060:"a4a4b44f",73249:"6b863ffe",73419:"78729bc5",73443:"f064c580",73602:"68c9c4d9",73796:"52f85e79",74275:"04909339",74772:"fe2304b2",74963:"3f156112",75088:"4d3332b5",75488:"80bb8867",75586:"ced09747",75658:"39b67445",76258:"e4fa02c3",76300:"de74179d",76454:"392d130c",76515:"09d48160",76572:"844b09b1",76780:"9fcdbad9",76843:"1987905e",77225:"d02aa824",77390:"a83939c0",77491:"28d99640",77682:"dd07edcb",77697:"8ab49adf",77969:"f101a58f",78047:"17c85fed",78480:"6094a26e",78579:"29367017",78817:"861d7487",79043:"9386a8fc",79079:"550ded59",79088:"4dea5ba6",79328:"01e88565",79362:"0185cc0f",80053:"57f575fd",80112:"fd699d0f",80322:"c2a5fb58",80598:"9604dd74",80707:"693fd0ce",80751:"17b0c135",80829:"a2ebc88c",80837:"842e3aa3",80874:"bf4d4db4",80982:"3a6c096a",81102:"ddb287b3",81340:"c70c4ed4",81663:"16985740",81787:"e94876f2",81878:"5025bbc2",81913:"45c6b533",81970:"f817585c",82158:"34c7f206",82412:"06ef2967",82569:"b8197956",82643:"aab4adeb",82668:"86174e6f",82681:"431ff015",82723:"2533d7d1",82744:"251a5009",82880:"e515703a",82994:"db964a35",83251:"5b905490",83378:"30d16d45",83393:"b4209ab4",83669:"b09b75c1",83781:"40717c05",83790:"eaac0795",84202:"3d5cb62e",84207:"75958044",84298:"a9f205e4",84880:"ee402f90",84908:"2baaa7ee",84954:"a71c1740",84979:"2a25ce30",85043:"f98b3ebd",85122:"e3efa00f",85332:"4a6b1343",85433:"81df0347",85648:"e46388c0",85729:"0a998fbe",85847:"0064d3f3",86184:"ce1a31c9",86248:"41094952",86482:"79b92687",86704:"4f81ddb2",86856:"985584fd",87320:"e9b88331",87384:"6d8d8c86",87452:"b3e364f6",87597:"41108fd6",87684:"e7b3125d",87780:"f43ae5ba",87808:"37b0acfa",88003:"4b4a7a4c",88838:"1a748968",88984:"3ff08287",89220:"e11e9cc5",89262:"0e62ec09",89407:"e589fd66",89450:"f36475fd",89461:"2a186faa",89463:"fcc8cf1f",89767:"5f64aaac",89864:"da57dee1",90288:"f854e6db",90544:"ebad36c0",91078:"8a9dd085",91103:"634f6370",91210:"f250c9ac",91214:"9600caec",91387:"50b3c0bf",91391:"6e8550b2",91570:"c5bc4d89",91747:"1df1ce06",91755:"efaa05fe",91824:"d2381236",91871:"264fce60",91895:"7233fcae",91921:"0f73e2cb",92018:"18420d4b",92045:"b7082472",92059:"3cb43b94",92321:"a13df7c4",92805:"bc260ba1",92836:"e6ce93fd",92867:"86a6b9e8",92982:"902b0046",93089:"5241fd92",93377:"aa0a6603",94330:"c6afff6b",94353:"a5690af9",94395:"8bb62ead",94508:"a7b3b267",94548:"7223c440",94553:"49648640",94579:"17f5fc3c",94597:"985993f6",94603:"107f7123",94623:"552eed53",94702:"239c52b1",94744:"1273506a",94756:"e8ca3715",94916:"ba81f68e",94994:"e5301c26",95049:"1d1e73a8",95064:"a0f894d7",95521:"64a26515",95584:"9c262440",95602:"279c2c7b",95717:"2a4b9c2d",95768:"92ce7313",95974:"53f9c073",96350:"146afc64",96673:"1c5fb002",96816:"d0dd057b",96960:"05a860df",97030:"47d88247",97065:"6050493a",97165:"c0ed39cb",97297:"2ae26e66",97381:"3bd2df9d",97715:"61cec5c2",97721:"489ca716",97800:"28867a86",97920:"b813a053",97988:"956d5ccf",98050:"0cdbfd5b",98197:"69b26a94",98462:"9f99c59e",98528:"f7d9cf13",98546:"238da816",98583:"1cb4799b",99119:"8e792955",99128:"abb2247b",99147:"2e0f9fd2",99161:"3e7e608f",99374:"4be47182",99378:"e5729889",99601:"3a075b02",99781:"0dcdc1ac",99970:"5f073138"}[e]+".js",r.miniCssF=e=>{},r.g=function(){if("object"==typeof globalThis)return globalThis;try{return this||new Function("return this")()}catch(e){if("object"==typeof window)return window}}(),r.o=(e,b)=>Object.prototype.hasOwnProperty.call(e,b),d={},a="synapseml:",r.l=(e,b,f,c)=>{if(d[e])d[e].push(b);else{var t,o;if(void 0!==f)for(var n=document.getElementsByTagName("script"),i=0;i<n.length;i++){var u=n[i];if(u.getAttribute("src")==e||u.getAttribute("data-webpack")==a+f){t=u;break}}t||(o=!0,(t=document.createElement("script")).charset="utf-8",t.timeout=120,r.nc&&t.setAttribute("nonce",r.nc),t.setAttribute("data-webpack",a+f),t.src=e),d[e]=[b];var l=(b,f)=>{t.onerror=t.onload=null,clearTimeout(s);var a=d[e];if(delete d[e],t.parentNode&&t.parentNode.removeChild(t),a&&a.forEach((e=>e(f))),b)return b(f)},s=setTimeout(l.bind(null,void 0,{type:"timeout",target:t}),12e4);t.onerror=l.bind(null,t.onerror),t.onload=l.bind(null,t.onload),o&&document.head.appendChild(t)}},r.r=e=>{"undefined"!=typeof Symbol&&Symbol.toStringTag&&Object.defineProperty(e,Symbol.toStringTag,{value:"Module"}),Object.defineProperty(e,"__esModule",{value:!0})},r.p="/SynapseML/",r.gca=function(e){return e={15516451:"99128",17896441:"27918",20641120:"787",26631026:"78480",55203621:"3554",71042800:"59484",75406112:"73060",75801678:"6289",90742975:"36005","32fe2e34":"226",e8d66fc1:"271","88b63415":"360","001aea4c":"390","7b60d8e3":"403",c4d68582:"429",b83021be:"587",b66ab6b5:"688",b3272230:"806","73c2022c":"847","0a65b9de":"939","421d486c":"1084","1f6f5ac9":"1123","255635fc":"1172",f264933a:"1283",fd4bd09d:"1286","3cbfbf75":"1773","513002de":"1802",ad10988d:"1854","368de161":"1985","6c82e982":"2070",b72abe57:"2195","59e4d88e":"2381","09bc1516":"2408",e4e95844:"2797","980b00ea":"2901","6c61f93b":"3202","6e159789":"3611","513214d5":"3701",d1428557:"3765",b41f295c:"3814","445012ce":"3871","60a2189a":"3953","73cf52e8":"4026","1ad91055":"4193",ea2d2bcc:"4204","1e0d2d3b":"4503",c065fcb9:"4800",cf57716c:"5000","405fabd0":"5253","33b3776b":"5254",b8963dc0:"5287",fc338b02:"5378","1155dc6f":"5789","78cb5959":"5909","10b7acf6":"6004","5e008bd1":"6304","4bbbdfcf":"6413","707d2a35":"6633",ed40d5f1:"6712","567cd2fd":"7201","268b7b40":"7234","9530a2cf":"7509","9c5088a2":"7925","9da9112f":"8006",e24ee294:"8040",d13e532e:"8146",e7bd83ca:"8168","2d527871":"8749",a563fa93:"8839","41f327fc":"8966","1518f45a":"9296","4f1ca6a9":"9540","014ce9a3":"10643",e12b4691:"10912",ac581902:"10978",c76e8cac:"11298",fba72e87:"11542","2ece0fbe":"11561","3b5de274":"11977",d05981ec:"12124","6c7ccb6d":"12398","5962ef6e":"12630","75514f6a":"12824","4bea8531":"12840","1b3ab3b8":"13431",ef6ec597:"13482","1b7af47d":"14273",a5ffe972:"14341","1842239f":"14535","90e4432b":"15095","603706f6":"15149","4f81391a":"15165",b7787e72:"15253","1abf1e18":"15343",ac39f7dc:"15404","3a4f9d93":"15487",a9330763:"15511",ebf648dd:"15566",dd29718b:"15579",f3e986ee:"15788","18c7d2a3":"15800","07f6b90a":"15839","0f113696":"16030","76d23901":"16269",bef0d2d3:"16286","3c1f4383":"16551","5b779334":"16833","51dbb02f":"16975","3ea19ec0":"17026","6eddee4c":"17295",ad139e3c:"17538","66d63bfc":"17542",b1706689:"17655",b328a361:"17782","1766722a":"17817",dd3f9ada:"17820","63cd8b22":"17973","445a1b28":"18373",ceeda143:"18443",d84904f9:"18563",c73cfc52:"18825","514f485e":"18828","11c317cb":"18933",c14fdd92:"19042","8bff2444":"19336","548964ce":"19465","15ea8379":"19483","2b30bc14":"19603",b2be3901:"19775","275c0efd":"19950","20e0fe38":"21102","6e4e5cd9":"21401",cb7c2a83:"21933",a34c49d0:"22007",e316de39:"22011","5407376f":"22287",aa5548ab:"22341","8af72580":"22577","49e6864b":"22707","6b40ec54":"22712","88f8e3e1":"22970","052105dd":"23036",c0391845:"23234","4e3910af":"23544",f21c13c4:"23807","92ce4fd5":"23838","35826f14":"23894","2bcda99e":"23941","48329b94":"23979","7aeb6f16":"24046",c5c95e45:"24422","1a3608a9":"24484",b962e007:"24594",e579bad6:"24627","10e13725":"25116",cf666d49:"25449",cee4c3b9:"25844","38d6824e":"25990","295a8e78":"26021","0eaf8468":"26184","03d0d544":"26676",c2e57a1b:"26685","8a1bca9d":"26871",ce40236b:"27497",a07d09c1:"27572","35bdc661":"27813",dc1e40d7:"27991","8d990105":"28456","6e2678b7":"28465",dac82c00:"28559",f1b2dc7f:"28634",b066233f:"28696",d620a661:"28746","6e71bda3":"28955","2d5b0e6a":"28957","9783ff24":"29006","7c007ec6":"29283","1be78505":"29514","905ce8b5":"29535",c82134d8:"29540","166d99f5":"29696",b9f125cc:"29806","39a50b22":"29845","5eddefdd":"30013",b739c60a:"30023","627aefca":"30301","22444eb9":"30335","0a9c6347":"30460","47bbce76":"30464",a6191053:"30927",c95882d3:"31153",fd3b6ea5:"31279","521f9727":"31333","645abc7d":"31836","9fa7e8eb":"31896","561bd03d":"32185","5f17911b":"32599","8ff6772d":"32777","0606f1a5":"32789","5bf420e9":"32855",da508cef:"33064","6a26e359":"33195","5c18deb5":"33358","90f00051":"33631","01587ade":"33988","141dc4a8":"34334","3e1e84da":"34397",e887b273:"34654","0d432c84":"34797","7275f11a":"34940","2c5f0abd":"35015","0daa5b3f":"35024","767a7177":"35040",c3b1d949:"35056","061b38c5":"35124","54287b47":"35328","22f921cc":"35517",c3dac62d:"35537","0637d1e7":"35750","0453ae68":"35939","50af03e4":"36204","6c7d288d":"36224",b0e28723:"36285","589adaf2":"36572",b7226801:"36585",affaa541:"36673",aba932dd:"36698","5ef28062":"36746",b4ae214b:"36773",df01388f:"36828",f027bb15:"36916","1e57c92a":"36958","83586bff":"37001","40b49758":"37069","27de0303":"37334",ef77ef39:"37392","21f47424":"37474",a406e231:"37486","0713a501":"37592","077ca2ec":"37610",d57515f7:"37638","4631a032":"37676","5efd5006":"37742",d9e5e528:"38057","6f8cd013":"38078","012c1c0f":"38261","70b306ba":"38377","327be84b":"38399","16d05917":"38759","5e7c3303":"38781",bf04857e:"38846",b7802ae1:"38859","4b18e5ae":"38962",ae89cd3b:"39139",bb7a48da:"39242","0d6d64cd":"39307","16d8ca91":"39402","4e105cba":"39596","2e93ae0f":"40023",fece3c24:"40367","9e5b788f":"40412","610d430a":"40659","965ed185":"40852","1db1f785":"40950","04804d02":"41607",b66e2100:"41719","391cb159":"41867","79114dfe":"41944","2cec6bcf":"41987","328f3af4":"42037",e3089280:"42283","1789daeb":"42377","31ee0fa3":"42428","6c4c8509":"42522",e21a3367:"42638","3ee83f58":"42802","993c5022":"42815","423540f2":"42852","42736d5f":"42947",de3bb7c3:"42970","1587ba1d":"43156","455aa1bd":"43251","995576e9":"43344","494501bd":"43426","336404dc":"43488","034e6169":"43730",d41234a3:"44177","70fe7dd7":"44178","00ab2cda":"44334",cd8e97a5:"44422",d81d1137:"44490","99e5ffa1":"44504","2d8a44d7":"44534",fe6131e8:"44539","104b9b56":"45282","7649ec47":"45370",bc85b804:"45503","7ad384dc":"45589","8799ef55":"45938",ccc49370:"46103","96cf5ff0":"46135","0fb98d33":"46338","98258c5e":"46559",b4898d44:"46613",c39e3602:"46807","79d79762":"46845",c5775233:"47454","01eb996b":"47552","408524d4":"48007","59ef8022":"48491","25fb872c":"48602","13246fd5":"48807",eed630e0:"49101",efbc69e1:"49196","3e7c5569":"49778","3a7a88af":"49814",fc598b5d:"50002",a81cf8b0:"50040","9c279ae7":"50196",fe2c893b:"50297","97592aac":"50653","069e3c4c":"50745","5c0b440a":"50783",ff428354:"51078","6b9bdd6f":"51105","6a9ad78d":"51317",b4a95996:"51478",e9aab890:"51732",d8d6ba90:"51842","644ff257":"51910","9688a5be":"51918",c4d09a44:"52406","345903d4":"52433","2e5f4263":"52447","814f3328":"52535",a3fd5a6d:"52730","9b020c4a":"52752",ed5c6c48:"52787","12b89d4a":"52804",bf002efe:"53017","943afd32":"53050","86ed1ff2":"53090",b88f1f9b:"53338",a8456d9f:"53572","9e4087bc":"53608",cb5f1a06:"54528",f537d1e9:"54890","76b137bf":"55276",ca0a1bad:"55305",bece3771:"55538","41ee0cff":"55702",ce7af6ae:"55864",cff2e56e:"56373","1293b2b0":"57074",d9837698:"57168","9d095dba":"57231",e2689dfc:"57258","39122aab":"57689",be95fd8f:"57887","15a59c33":"57924",d39aa6d3:"57930","4d31bfb3":"58092",d989c707:"58462","0349e4f0":"58524",b940e0b5:"58738","990f07de":"58755","3bed3e2f":"58846","7c2e1100":"58861",dfb6dfe9:"59076","4801d370":"59394","76bf568a":"59823","403e642d":"60055",f322d2da:"60445",f0649d54:"60880","3ef20ed6":"61407",b5375b6f:"61428",b75118f0:"61442","04fc65cc":"61755",a0d8a1a2:"61778",f38a0cbd:"61829","5cfa133c":"62079","2c98ca3e":"62106","04ee4ab4":"62217",ddacd194:"62465",c38c658c:"62565","4ef2f7cf":"62675",d95a70ea:"62721","52880d18":"62750","08019edc":"62773",bf8a5c8a:"63002","95d9e891":"63099","74091a62":"63196","56543a15":"63402",ed6d544d:"63448","3bbe0f34":"64172",c4f5d8e4:"64195",b7c6295c:"64233","0e3f998d":"64482",d355047f:"64713",fa2e2688:"64927","37ab9beb":"64992","44ff0d07":"65437",c3c516ff:"65752","900b935c":"65942","74659d33":"66167",e5fe80e3:"66419","0fa36e32":"66555","8462374b":"66808","296054a7":"66945",e863531f:"66981",ba9c0924:"67218","99f8fee5":"67287","31bb7274":"67394","888e2229":"67448",a608660b:"67597",faca745a:"67664",db170a23:"67706",ed316aed:"67870",a1824f78:"68070",e84755e1:"68283","104bf0a6":"68314","3201258e":"68526",c991f47b:"68549","18c9c2ba":"68585",fd748117:"68809","57e687e8":"69159","8181e18a":"69226","232d09b0":"69314","3fb29942":"69559",d1246f49:"69966","77c259dd":"70224","7aa7b8e0":"70344","6a07b55e":"70406","2461af4d":"70880","1a215693":"71535","179d57fa":"71580",a877f9e5:"72141","38b375cc":"72249","3d963cd5":"72286","56087ddb":"72293","1ef65056":"72326","13f17286":"72912","5225b7e0":"73249","8e32e44d":"73419","91e9cb67":"73443",af1e70b6:"73602","6a95f87e":"73796","792eae7e":"74275","4997ef4f":"74772","6f479459":"74963","7a1d9beb":"75088",cbc3190d:"75488",e7250a93:"75586",c23b5e26:"75658","3dafa002":"76258",d252041b:"76300","34f00221":"76454","9cb6fa7c":"76515","58486a98":"76572","62983deb":"76843","51878b77":"77225",b35e9998:"77390",d3093636:"77491","50f80512":"77682","39582d99":"77697","5887dc62":"77969","3b75a1c6":"78047","0b416bde":"78579","88e44e99":"78817","00353f68":"79043",efc1978d:"79079","0a954229":"79088",a0434473:"79328","34ff21b6":"79362","935f2afb":"80053","8fd0c721":"80112",b576c645:"80322",e1269457:"80598",ab9a8589:"80707","556ad472":"80751",ee6cea3b:"80829","17f418b7":"80837",b92ffdc6:"80874","2fb6c1b0":"80982",d25cde70:"81102",e1b2b5f3:"81340","00056db7":"81663",e4ba4487:"81787",fa889bee:"81878","1d820955":"81913","01a15f20":"81970",f7580424:"82158","12d85ab5":"82412",e03f5a54:"82569",f228fdc5:"82643","3a1147b5":"82668","62ccfbde":"82681","23af8e3e":"82723",f8597cb0:"82744",e1ed1e14:"82880","48cf7354":"82994",c34b1e36:"83251",fbb011bb:"83378","3e9a7422":"83393",d707d2ed:"83669","866b91a1":"83781","604b2d50":"83790",c2492d56:"84202",f14633e7:"84207","6094981e":"84298",f8aab3ca:"84880","59d9fecc":"84908",f7a32432:"84954",e8b9c2c7:"84979","8eb438b4":"85043",c2e40c34:"85122","321ba9c5":"85332",ffb7f88a:"85433",ee83122d:"85648","56d039a1":"85729","4e044b4e":"85847","4130db01":"86184","7bf16cc6":"86248","3c22aa66":"86482","67e06a8b":"86704",b80b493a:"86856",c40984d1:"87320",b35d004b:"87384","93ba15e0":"87452","66fcb786":"87597",b38eec89:"87684",b28fe732:"87808","5066efb2":"88003","70e8008e":"88838",d535b358:"88984",a2b5e5df:"89220","120352d6":"89262",deef0af8:"89407","83203dd1":"89450","6ff7775f":"89461",a19ce767:"89463",c413b43a:"89767","1dc2f362":"89864","69bb4b11":"90288","4585b70e":"90544","80472ecc":"91078","3ac37a06":"91103",e2ce57ed:"91210",cc52e3cd:"91214",d36f3429:"91387","2137a7cd":"91391","4de42975":"91570","85aed2f9":"91747",bc4bf151:"91755",c90c942b:"91824","8590feaa":"91871","78aebd5d":"91895","8328d740":"91921",dc7f7ac1:"92018","0e0ee9b2":"92045",b33598ac:"92059","325e22bd":"92321",d175df5e:"92805","8a46fce0":"92836","5dbf4619":"92867","3647ac9b":"92982",a6aa9e1f:"93089","318ccb43":"93377",d8b3ac6a:"94330","1f31a275":"94353","0e2bf69e":"94395",e464d112:"94508","53a5cb1e":"94548",d7f58694:"94553","94d743d6":"94579",d0cb74df:"94597",f7bfff73:"94603",e402e26d:"94623","5a3e1bca":"94702","1014102e":"94744","4cee39d7":"94756","82878c84":"94916","109f6864":"94994",f85db098:"95049",e9b75d67:"95064","283b8f99":"95521",bd1bcb2c:"95584","57ef17c5":"95602",d0c5e4fa:"95717",a34b09d4:"95768","8cb11643":"95974","1e14124f":"96350",e5c4ef01:"96673","82d7cc79":"96816","1a62d1af":"96960",c764f2c0:"97030",fe5d19cb:"97065","294c060e":"97165",e0077969:"97297","4db82253":"97381","2a023663":"97715","0e16e9ea":"97721","09990b79":"97800","1a4e3797":"97920","4e432063":"97988",d3c9b0d7:"98050",eb0ef952:"98197",bc6a9944:"98462","7296bed0":"98528",f5b4e517:"98546","6bdbf6b8":"98583","2406587d":"99119","7aa49360":"99147","69dc6d0a":"99161","58b8f176":"99374",efbb59fb:"99378",fc47fc53:"99601",dea122e2:"99781","093b81f4":"99970"}[e]||e,r.p+r.u(e)},(()=>{var e={51303:0,40532:0};r.f.j=(b,f)=>{var d=r.o(e,b)?e[b]:void 0;if(0!==d)if(d)f.push(d[2]);else if(/^(40532|51303)$/.test(b))e[b]=0;else{var a=new Promise(((f,a)=>d=e[b]=[f,a]));f.push(d[2]=a);var c=r.p+r.u(b),t=new Error;r.l(c,(f=>{if(r.o(e,b)&&(0!==(d=e[b])&&(e[b]=void 0),d)){var a=f&&("load"===f.type?"missing":f.type),c=f&&f.target&&f.target.src;t.message="Loading chunk "+b+" failed.\n("+a+": "+c+")",t.name="ChunkLoadError",t.type=a,t.request=c,d[1](t)}}),"chunk-"+b,b)}},r.O.j=b=>0===e[b];var b=(b,f)=>{var d,a,c=f[0],t=f[1],o=f[2],n=0;if(c.some((b=>0!==e[b]))){for(d in t)r.o(t,d)&&(r.m[d]=t[d]);if(o)var i=o(r)}for(b&&b(f);n<c.length;n++)a=c[n],r.o(e,a)&&e[a]&&e[a][0](),e[a]=0;return r.O(i)},f=self.webpackChunksynapseml=self.webpackChunksynapseml||[];f.forEach(b.bind(null,0)),f.push=b.bind(null,f.push.bind(f))})()})();
\ No newline at end of file
diff --git a/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/index.html b/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/index.html
index c279dc9ff1..d13ba490d3 100644
--- a/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/index.html	
+++ b/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">Publication - Flexible and Scalable Deep Learning with MMLSpark</h1><div class="container_mt6G margin-vert--md"><time datetime="2018-04-01T00:00:00.000Z" itemprop="datePublished">April 1, 2018</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><p>In this work, we detail a novel open source library called MMLSpark that combines the flexible deep learning library Cognitive Toolkit with the distributed computing framework Apache Spark. To achieve this union, we have contributed Java Language bindings to the Cognitive Toolkit <!-- --> and added several new components to the Spark ecosystem. In addition, we integrate OpenCV, the popular image processing library, with Spark. We present a tool that automatically generates PySpark wrappers from any SparkML estimator, and we use this tool to expose all work to the PySpark ecosystem. Finally, we provide a large library of tools for working and developing within the Spark ecosystem. We apply this work to the automated classification of Snow Leopards from camera trap images, and provide an end-to-end solution for the non-profit conservation organization, the Snow Leopard Trust.</p><p><a href="https://www.microsoft.com/en-us/research/publication/flexible-and-scalable-deep-learning-with-mmlspark/" target="_blank" rel="noopener noreferrer">Read More</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/index.html b/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/index.html
index fac721d2ba..b40ef20693 100644
--- a/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/index.html	
+++ b/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a aria-current="page" class="sidebarItemLink_mo7H sidebarItemLinkActive_I1ZP" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</h1><div class="container_mt6G margin-vert--md"><time datetime="2019-06-01T00:00:00.000Z" itemprop="datePublished">June 1, 2019</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><p>We introduce Microsoft Machine Learning for Apache Spark (MMLSpark), an ecosystem of enhancements that expand the Apache Spark distributed computing library. The library tackles problems in Deep Learning, Micro-Service Orchestration, Gradient Boosting, Model Interpretability, and other areas of modern computation. <!-- --> Furthermore, we present a novel system called Spark Serving that allows users to run any Apache Spark program as a distributed, submillisecond latency web service backed by their existing Spark Cluster. All MMLSpark contributions have the same API to enable simple composition across frameworks and usage across batch, streaming, and RESTful web serving scenarios on static, elastic, or serverless clusters. We showcase MMLSpark by creating a method for deep object detection capable of learning without human labeled data and demonstrate its effectiveness for Snow Leopard conservation.</p><p><a href="https://www.microsoft.com/en-us/research/publication/mmlspark-unifying-machine-learning-ecosystems-at-massive-scales/" target="_blank" rel="noopener noreferrer">Read More</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">Dear Spark developers: Welcome to Azure Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Publication - Flexible and Scalable Deep Learning with MMLSpark</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/2019/08/24/Welcome to Azure Cognitive Services/index.html b/blog/2019/08/24/Welcome to Azure Cognitive Services/index.html
index aaeb3c1402..bae5db0bfc 100644
--- a/blog/2019/08/24/Welcome to Azure Cognitive Services/index.html	
+++ b/blog/2019/08/24/Welcome to Azure Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a aria-current="page" class="sidebarItemLink_mo7H sidebarItemLinkActive_I1ZP" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">Dear Spark developers: Welcome to Azure Cognitive Services</h1><div class="container_mt6G margin-vert--md"><time datetime="2019-08-24T00:00:00.000Z" itemprop="datePublished">August 24, 2019</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><p>&quot;Integrating the power of Azure Cognitive Services into your big data workflows on Apache Spark™</p><p>Today at Spark + AI Summit 2019, we&#x27;re excited to introduce a new set of models in the SparkML ecosystem that make it easy to use Azure Cognitive Services at terabyte scales. <!-- --> With only a few lines of code, developers can embed cognitive services within their distributed machine learning pipelines in Spark ML. Additionally, these contributions allow Spark users to chain or Pipeline services together with deep networks, gradient boosted trees, and any SparkML model and apply these hybrid models in elastic and serverless distributed systems.</p><p>From image recognition to object detection using speech recognition, translation, and text-to-speech, Azure Cognitive Services makes it easy for developers to add intelligent capabilities to their applications in any scenario. To this date, more than a million developers have already discovered and tried Cognitive Services to accelerate breakthrough experiences in their application.</p><p><a href="https://azure.microsoft.com/en-us/blog/dear-spark-developers-welcome-to-azure-cognitive-services/" target="_blank" rel="noopener noreferrer">Read More</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">MMLSpark: empowering AI for Good with Mark Hamilton</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/index.html b/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/index.html
index cda6bbab70..fa4f08b250 100644
--- a/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/index.html	
+++ b/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a aria-current="page" class="sidebarItemLink_mo7H sidebarItemLinkActive_I1ZP" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">MMLSpark: empowering AI for Good with Mark Hamilton</h1><div class="container_mt6G margin-vert--md"><time datetime="2019-10-02T00:00:00.000Z" itemprop="datePublished">October 2, 2019</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><img loading="lazy" src="https://www.microsoft.com/en-us/research/uploads/prod/2019/09/Mark-Hamilton_Podcast_Site_09_2019_1400x788.png" class="img_ev3q"><h4>Episode 92, October 2, 2019</h4><p>If someone asked you what snow leopards and Vincent Van Gogh have in common, you might think it was the beginning of a joke. <!-- --> It’s not. but if it was, Mark Hamilton, budding PhD student, member of Microsoft’s Cognitive Services group and frequent Microsoft Research collaborator, would tell you the punch line is machine learning. More specifically, Microsoft Machine Learning for Apache Spark (MMLSpark for short), a powerful yet elastic open source machine learning library that’s finding its way beyond business and into “AI for Good” applications such as the environment and the arts.</p><p>Today, Mark talks about his love of mathematics and his desire to solve big, crazy, core knowledge-sized problems. He tells us all about MMLSpark and how it’s being used by organizations like the Snow Leopard Trust and the Metropolitan Museum of Art. And he reveals how the persuasive advice of a really smart big sister helped launch an exciting career in AI research and development.</p><p><a href="https://www.microsoft.com/en-us/research/podcast/mmlspark-empowering-ai-for-good-with-mark-hamilton/" target="_blank" rel="noopener noreferrer">Read More</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">Publication - Large-Scale Intelligent Microservices</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Dear Spark developers: Welcome to Azure Cognitive Services</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/2020/12/01/Large-Scale Intelligent Microservices/index.html b/blog/2020/12/01/Large-Scale Intelligent Microservices/index.html
index 62033f9c6c..a4be9d5250 100644
--- a/blog/2020/12/01/Large-Scale Intelligent Microservices/index.html	
+++ b/blog/2020/12/01/Large-Scale Intelligent Microservices/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a aria-current="page" class="sidebarItemLink_mo7H sidebarItemLinkActive_I1ZP" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">Publication - Large-Scale Intelligent Microservices</h1><div class="container_mt6G margin-vert--md"><time datetime="2020-12-01T00:00:00.000Z" itemprop="datePublished">December 1, 2020</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><p>The process of deploying Machine Learning (ML) algorithms within databases is challenging. The varied computational footprints of modern ML algorithms and the myriad of database technologies, each with their own restrictive syntax, make such tasks more than a little complex. We introduce an Apache Spark-based micro-service orchestration <!-- --> framework that extends database operations to include web service primitives. Our system can orchestrate web services across hundreds of machines and takes full advantage of cluster, thread, and asynchronous parallelism. Using this framework, we provide large scale clients for intelligent services such as speech, vision, search, anomaly detection, and text analysis. These clients allow users to integrate ready-to-use intelligence into any datastore with an Apache Spark connector. To eliminate much of the overhead from network communication, we also introduce a low-latency containerized version of our architecture. We demonstrate that the services we investigate are competitive on several important benchmarks. Finally, we present two applications of this framework to create intelligent search engines and real-time auto race analytics systems.</p><p><a href="https://www.microsoft.com/en-us/research/publication/large-scale-services/" target="_blank" rel="noopener noreferrer">Read More</a></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/blog/overview/"><div class="pagination-nav__sublabel">Newer Post</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">MMLSpark: empowering AI for Good with Mark Hamilton</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/archive/index.html b/blog/archive/index.html
index 597dedcf57..87a7b97668 100644
--- a/blog/archive/index.html
+++ b/blog/archive/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/archive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><header class="hero hero--primary"><div class="container"><h1 class="hero__title">Archive</h1><p class="hero__subtitle">Archive</p></div></header><main><section class="margin-vert--lg"><div class="container"><div class="row"><div class="col col--4 margin-vert--lg"><h3>2018</h3><ul><li><a href="/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/">April 1, 2018<!-- --> - <!-- -->Publication - Flexible and Scalable Deep Learning with MMLSpark</a></li></ul></div><div class="col col--4 margin-vert--lg"><h3>2019</h3><ul><li><a href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">October 2, 2019<!-- --> - <!-- -->MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li><a href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">August 24, 2019<!-- --> - <!-- -->Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li><a href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">June 1, 2019<!-- --> - <!-- -->Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></div><div class="col col--4 margin-vert--lg"><h3>2020</h3><ul><li><a href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">December 1, 2020<!-- --> - <!-- -->Publication - Large-Scale Intelligent Microservices</a></li></ul></div><div class="col col--4 margin-vert--lg"><h3>2021</h3><ul><li><a href="/SynapseML/blog/overview/">October 18, 2021<!-- --> - <!-- -->What is SynapseML?</a></li></ul></div></div></div></section></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/index.html b/blog/index.html
index d5ab53b6cf..64eef93443 100644
--- a/blog/index.html
+++ b/blog/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/overview/">What is SynapseML?</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2021-10-18T18:07:02.000Z" itemprop="datePublished">October 18, 2021</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p></div></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2020-12-01T00:00:00.000Z" itemprop="datePublished">December 1, 2020</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><p>The process of deploying Machine Learning (ML) algorithms within databases is challenging. The varied computational footprints of modern ML algorithms and the myriad of database technologies, each with their own restrictive syntax, make such tasks more than a little complex. We introduce an Apache Spark-based micro-service orchestration</p></div><footer class="row docusaurus-mt-lg"><div class="col text--right"><a aria-label="Read more about Publication - Large-Scale Intelligent Microservices" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/"><b>Read More</b></a></div></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2019-10-02T00:00:00.000Z" itemprop="datePublished">October 2, 2019</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><img loading="lazy" src="https://www.microsoft.com/en-us/research/uploads/prod/2019/09/Mark-Hamilton_Podcast_Site_09_2019_1400x788.png" class="img_ev3q"><h4>Episode 92, October 2, 2019</h4><p>If someone asked you what snow leopards and Vincent Van Gogh have in common, you might think it was the beginning of a joke.</p></div><footer class="row docusaurus-mt-lg"><div class="col text--right"><a aria-label="Read more about MMLSpark: empowering AI for Good with Mark Hamilton" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/"><b>Read More</b></a></div></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2019-08-24T00:00:00.000Z" itemprop="datePublished">August 24, 2019</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><p>&quot;Integrating the power of Azure Cognitive Services into your big data workflows on Apache Spark™</p><p>Today at Spark + AI Summit 2019, we&#x27;re excited to introduce a new set of models in the SparkML ecosystem that make it easy to use Azure Cognitive Services at terabyte scales.</p></div><footer class="row docusaurus-mt-lg"><div class="col text--right"><a aria-label="Read more about Dear Spark developers: Welcome to Azure Cognitive Services" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/"><b>Read More</b></a></div></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2019-06-01T00:00:00.000Z" itemprop="datePublished">June 1, 2019</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><p>We introduce Microsoft Machine Learning for Apache Spark (MMLSpark), an ecosystem of enhancements that expand the Apache Spark distributed computing library. The library tackles problems in Deep Learning, Micro-Service Orchestration, Gradient Boosting, Model Interpretability, and other areas of modern computation.</p></div><footer class="row docusaurus-mt-lg"><div class="col text--right"><a aria-label="Read more about Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/"><b>Read More</b></a></div></footer></article><article class="margin-bottom--xl" itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h2 class="title_f1Hy" itemprop="headline"><a itemprop="url" href="/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/">Publication - Flexible and Scalable Deep Learning with MMLSpark</a></h2><div class="container_mt6G margin-vert--md"><time datetime="2018-04-01T00:00:00.000Z" itemprop="datePublished">April 1, 2018</time> · <!-- -->One min read</div></header><div class="markdown" itemprop="articleBody"><p>In this work, we detail a novel open source library called MMLSpark that combines the flexible deep learning library Cognitive Toolkit with the distributed computing framework Apache Spark. To achieve this union, we have contributed Java Language bindings to the Cognitive Toolkit</p></div><footer class="row docusaurus-mt-lg"><div class="col text--right"><a aria-label="Read more about Publication - Flexible and Scalable Deep Learning with MMLSpark" href="/SynapseML/blog/2018/04/01/Flexible and Scalable Deep Learning with MMLSpark/"><b>Read More</b></a></div></footer></article><nav class="pagination-nav" aria-label="Blog list page navigation"></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/blog/overview/index.html b/blog/overview/index.html
index 91b941ae0e..7a99b7fb3b 100644
--- a/blog/overview/index.html
+++ b/blog/overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/blog/overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><div class="row"><aside class="col col--3"><nav class="sidebar_re4s thin-scrollbar" aria-label="Blog recent posts navigation"><div class="sidebarItemTitle_pO2u margin-bottom--md">Recent posts</div><ul class="sidebarItemList_Yudw clean-list"><li class="sidebarItem__DBe"><a aria-current="page" class="sidebarItemLink_mo7H sidebarItemLinkActive_I1ZP" href="/SynapseML/blog/overview/">What is SynapseML?</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/">Publication - Large-Scale Intelligent Microservices</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/10/02/MMLSpark empowering AI for Good with Mark Hamilton/">MMLSpark: empowering AI for Good with Mark Hamilton</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/08/24/Welcome to Azure Cognitive Services/">Dear Spark developers: Welcome to Azure Cognitive Services</a></li><li class="sidebarItem__DBe"><a class="sidebarItemLink_mo7H" href="/SynapseML/blog/2019/06/01/MMLSpark Unifying Machine Learning Ecosystems at Massive Scales/">Publication - MMLSpark: Unifying Machine Learning Ecosystems at Massive Scales</a></li></ul></nav></aside><main class="col col--7" itemscope="" itemtype="http://schema.org/Blog"><article itemprop="blogPost" itemscope="" itemtype="http://schema.org/BlogPosting"><header><h1 class="title_f1Hy" itemprop="headline">What is SynapseML?</h1><div class="container_mt6G margin-vert--md"><time datetime="2021-10-18T18:07:02.000Z" itemprop="datePublished">October 18, 2021</time> · <!-- -->One min read</div></header><div id="__blog-post-container" class="markdown" itemprop="articleBody"><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Blog post page navigation"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/blog/2020/12/01/Large-Scale Intelligent Microservices/"><div class="pagination-nav__sublabel">Older Post</div><div class="pagination-nav__label">Publication - Large-Scale Intelligent Microservices</div></a></nav></main></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Deploy Models/Overview/index.html b/docs/0.11.3/Deploy Models/Overview/index.html
index d5ac3a2e38..0f02be503c 100644
--- a/docs/0.11.3/Deploy Models/Overview/index.html	
+++ b/docs/0.11.3/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/index.html
index c0c75fe8bf..a259db58a6 100644
--- a/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 522bc34c9d..33d73873f5 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Cognitive Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(&quot;cognitive-api-key&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the cognitive service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.text import TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The cognitive services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.document.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.text import TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(&quot;cognitive-api-key-2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/index.html
index a59ff7f8c7..d6c41d2a21 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/en-us/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/en-us/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/en-us/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/big-data/getting-started#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/en-us/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/en-us/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/en-us/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index b1d3291912..853b516e92 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Cognitive Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure Cognitive Services on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or timeseries, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure Cognitive Services, and we then use to the model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors. </p><p>To learn more about the Anomaly Detector Cognitive Service, refer to <a href="https://docs.microsoft.com/azure/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">item</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> item </span><span class="token keyword">in</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Overview/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Overview/index.html
index c2d5585f07..151be6a549 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Cognitive Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><p><a href="https://azure.microsoft.com/services/cognitive-services/" target="_blank" rel="noopener noreferrer">Azure Cognitive Services</a> are a suite of APIs, SDKs, and services available to help developers build intelligent applications without having direct AI or data science skills or knowledge by enabling developers to easily add cognitive features into their applications. The goal of Azure Cognitive Services is to help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure Cognitive Services can be categorized into five main pillars - Vision, Speech, Language, Web Search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/text-analytics/" target="_blank" rel="noopener noreferrer"><strong>Text Analytics</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.anomaly.html#module-synapse.ml.cognitive.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.anomaly.html#module-synapse.ml.cognitive.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/cognitive-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.bing.html#module-synapse.ml.cognitive.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/scala/com/microsoft/azure/synapse/ml/cognitive/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cognitive.search.html#module-synapse.ml.cognitive.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/cognitive-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Cognitive Services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general Cognitive Services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics-sample">Text Analytics sample<a href="#text-analytics-sample" class="hash-link" aria-label="Direct link to Text Analytics sample" title="Direct link to Text Analytics sample">​</a></h2><p>The <a href="https://azure.microsoft.com/services/cognitive-services/text-analytics/" target="_blank" rel="noopener noreferrer">Text Analytics</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.document.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics-for-health-sample">Text Analytics for Health Sample<a href="#text-analytics-for-health-sample" class="hash-link" aria-label="Direct link to Text Analytics for Health Sample" title="Direct link to Text Analytics for Health Sample">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/cognitive-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured texts such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator-sample">Translator sample<a href="#translator-sample" class="hash-link" aria-label="Direct link to Translator sample" title="Direct link to Translator sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure Cognitive Services family of cognitive APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used for text translation with any operating system. In this sample, we do a simple text translation by providing the sentences you want to translate and target languages you want to translate to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-sample">Form Recognizer sample<a href="#form-recognizer-sample" class="hash-link" aria-label="Direct link to Form Recognizer sample" title="Direct link to Form Recognizer sample">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. Identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more. In this sample, we analyze a business card image and extract its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer">Computer Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions. In this sample, we tag a list of images. Tags are one-word descriptions of things in the image like recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query. In this sample, we use a text query that looks for images with quotes. It returns a list of image URLs that contain photos related to our query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text-sample">Speech-to-Text sample<a href="#speech-to-text-sample" class="hash-link" aria-label="Direct link to Speech-to-Text sample" title="Direct link to Speech-to-Text sample">​</a></h2><p>The <a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. In this sample, we transcribe one audio file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/text-to-speech/#overview" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows one to build apps and services that speak naturally, choosing from more than 270 neural voices across 119 languages and variants.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detector-sample">Anomaly Detector sample<a href="#anomaly-detector-sample" class="hash-link" aria-label="Direct link to Anomaly Detector sample" title="Direct link to Anomaly Detector sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. In this sample, we use the service to find anomalies in the entire time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="arbitrary-web-apis">Arbitrary web APIs<a href="#arbitrary-web-apis" class="hash-link" aria-label="Direct link to Arbitrary web APIs" title="Direct link to Arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#text-analytics-sample" class="table-of-contents__link toc-highlight">Text Analytics sample</a></li><li><a href="#text-analytics-for-health-sample" class="table-of-contents__link toc-highlight">Text Analytics for Health Sample</a></li><li><a href="#translator-sample" class="table-of-contents__link toc-highlight">Translator sample</a></li><li><a href="#form-recognizer-sample" class="table-of-contents__link toc-highlight">Form Recognizer sample</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#speech-to-text-sample" class="table-of-contents__link toc-highlight">Speech-to-Text sample</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#anomaly-detector-sample" class="table-of-contents__link toc-highlight">Anomaly Detector sample</a></li><li><a href="#arbitrary-web-apis" class="table-of-contents__link toc-highlight">Arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index a7eb2914a0..683cfe0e23 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Cognitive Services on Spark</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index 5589842615..f7c59d8f7c 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="theme-doc-markdown markdown"><h1>Cognitive Services - Analyze Text</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Creare a Visual Search Engine</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html
index 4e854bcb1c..c6f79f281d 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Creare a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/en-us/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/en-us/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index d6a33918c9..f576e8b750 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your cognitive service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Creare a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index 681678de46..2ed739b07a 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/cognitive-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/cognitive-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install langchain openai</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index bb4fd18b25..0b414c468f 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index 14e6271a93..83307dc9a3 100644
--- a/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Cognitive Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Cognitive Services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/en-us/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure Cognitive Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account-cli?tabs=linux" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(&quot;anomaly-api-key&quot;)  # Paste your anomaly detector key here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location = &quot;westus2&quot;  # Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals = spark.read.csv(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-cognitive-services-on-spark">Run anomaly detection using Cognitive Services on Spark<a href="#run-anomaly-detection-using-cognitive-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using Cognitive Services on Spark" title="Direct link to Run anomaly detection using Cognitive Services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import col, struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive import SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.spark import FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(location)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setGroupbyCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSensitivity(95)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setGranularity(&quot;secondly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals.where(col(&quot;unitSymbol&quot;) == &quot;RPM&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;timestamp&quot;, col(&quot;dateTime&quot;).cast(&quot;string&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;value&quot;, col(&quot;measureValue&quot;).cast(&quot;double&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;grouping&quot;, struct(&quot;deviceId&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .mlTransform(detector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly.createOrReplaceTempView(&quot;df_anomaly&quot;)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly.select(&quot;timestamp&quot;, &quot;value&quot;, &quot;deviceId&quot;, &quot;anomalies.isAnomaly&quot;).show(3)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device = spark.sql(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">limit 200&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf = df_anomaly_single_device.toPandas()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset = df_anomaly_single_device.where(col(&quot;isAnomaly&quot;) == 1).toPandas()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.figure(figsize=(23, 8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedUpperValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;darkred&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=0.25,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;UpperMargin&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;darkgreen&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=2,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Expected Value&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;measureValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;b&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;royalblue&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=2,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Actual&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedLowerValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;black&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=0.25,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Lower Margin&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(adf_subset[&quot;timestamp&quot;], adf_subset[&quot;measureValue&quot;], &quot;ro&quot;, label=&quot;Anomaly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.legend()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.title(&quot;RPM Anomalies with Confidence Intervals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure Cognitive Services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-cognitive-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using Cognitive Services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 8f7533d5c0..72a1f7d094 100644
--- a/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/index.html b/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/index.html
index 14cd432fc5..fd5d4f0fc7 100644
--- a/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 8956121144..51d94642f4 100644
--- a/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 3c92e78316..d8be1d039f 100644
--- a/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index 1e97aba7f8..77780f80f2 100644
--- a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index eebe2210cf..4bb69d670a 100644
--- a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 9b7f2d2449..283480855c 100644
--- a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 34fc8ddde9..c82d84005f 100644
--- a/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 53bc124cbf..ce5bd303aa 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/index.html
index 3a104de8d9..57d8b1cd93 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:0.11.3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/index.html
index 520d99ce59..9404b553d7 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index 230dde0753..021943af60 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index 7f30a44f5d..fe7c7f4c51 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index f937dc6f8c..9e5add968a 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index e1e9ad8274..c882674c79 100644
--- a/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 488ff50dac..6ca1f018b9 100644
--- a/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index 08c2f40cf7..54b9f89ba7 100644
--- a/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/LightGBM/Overview/index.html b/docs/0.11.3/Explore Algorithms/LightGBM/Overview/index.html
index feac63911c..3e933a01c8 100644
--- a/docs/0.11.3/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/0.11.3/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index b0d00eab42..7e5a17cf18 100644
--- a/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/0.11.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/index.html b/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/index.html
index ae8054cbe8..ffefd8b9d6 100644
--- a/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai langchain pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six pytesseract unstructured</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index 6d879161c3..e8a54f971f 100644
--- a/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index 12734b4c3b..99981ce3a4 100644
--- a/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in Cognitive Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/OpenAI/index.html b/docs/0.11.3/Explore Algorithms/OpenAI/index.html
index b066014f68..ca571ca49d 100644
--- a/docs/0.11.3/Explore Algorithms/OpenAI/index.html	
+++ b/docs/0.11.3/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/index.html
index 4145d17c22..d1630204d4 100644
--- a/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index 1d0a5b584a..661b3ebf67 100644
--- a/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index 6d5672c8ea..384e389c4f 100644
--- a/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:0.11.3
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 525e840be5..53881fcd8c 100644
--- a/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index b4aa9b7379..780d4759d9 100644
--- a/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index 6b2212af10..d363f2677c 100644
--- a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index 8f4d535580..070aa5722c 100644
--- a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 4e2eb4053c..3cbff2bf75 100644
--- a/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index 92c12188fd..175552b0ea 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index f620e90c93..ade2cc28ce 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/index.html
index 1cd10afdbf..827cb870a8 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index 30983c3855..566d545082 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index cacaa6323c..cd00e753b3 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index c531d846fa..4ebd815ab0 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index 1d20caa7de..d3d12d7418 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index d266e4073e..b0049122d9 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 2ca84143b1..bcd85975db 100644
--- a/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array = udf(lambda vec: vec.toArray().tolist(), ArrayType(FloatType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vec2array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vec2array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index 54cfabd8e6..3d2f23f201 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index 236c850974..ae30aec97a 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index f792086041..dcee4b7b0e 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index 45623be335..717ef660d9 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index 09e106e71b..6a94d38edf 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 81acc7a17f..8e1450031d 100644
--- a/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Get Started/Create a Spark Cluster/index.html b/docs/0.11.3/Get Started/Create a Spark Cluster/index.html
index e373b8453d..c8624d60d5 100644
--- a/docs/0.11.3/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/0.11.3/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Get Started/Install SynapseML/index.html b/docs/0.11.3/Get Started/Install SynapseML/index.html
index 2e8de958ae..738e43d86e 100644
--- a/docs/0.11.3/Get Started/Install SynapseML/index.html	
+++ b/docs/0.11.3/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/0.11.3/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Get Started/Quickstart - Your First Models/index.html b/docs/0.11.3/Get Started/Quickstart - Your First Models/index.html
index 2d86df2b6f..586eb0702d 100644
--- a/docs/0.11.3/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/0.11.3/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure Cognitive Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-cognitive-services-to-transform-data-in-one-step">Use Cognitive Services to transform data in one step<a href="#use-cognitive-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Cognitive Services to transform data in one step" title="Direct link to Use Cognitive Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Cognitive Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-cognitive-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Cognitive Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Get Started/Set up Cognitive Services/index.html b/docs/0.11.3/Get Started/Set up Cognitive Services/index.html
index 7baf9c11e4..273ccbb768 100644
--- a/docs/0.11.3/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/0.11.3/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Cognitive Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Cognitive Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="cognitive-services">Cognitive Services<a href="#cognitive-services" class="hash-link" aria-label="Direct link to Cognitive Services" title="Direct link to Cognitive Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/products/cognitive-services/" target="_blank" rel="noopener noreferrer">Cognitive Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Cognitive Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create an Azure Cognitive multi-service (Decision, Language, Speech, Vision) resource</a>. Alternatively, you can follow the steps to <a href="https://learn.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account?tabs=decision%2Canomaly-detector%2Clanguage-service%2Ccomputer-vision%2Cwindows#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">create Single-service resource</a>. </li><li>Get your Cognitive Service resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#cognitive-services" class="table-of-contents__link toc-highlight">Cognitive Services</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Overview/index.html b/docs/0.11.3/Overview/index.html
index 51077a8b79..a83129d145 100644
--- a/docs/0.11.3/Overview/index.html
+++ b/docs/0.11.3/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/estimators/estimators_causal/index.html b/docs/0.11.3/Quick Examples/estimators/estimators_causal/index.html
index e341680503..691aeafcce 100644
--- a/docs/0.11.3/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/0.11.3/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/index.html b/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/index.html
index dd3e995944..034c8a98f5 100644
--- a/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/estimators/estimators_core/index.html b/docs/0.11.3/Quick Examples/estimators/estimators_core/index.html
index e169f8fff3..75b8b19972 100644
--- a/docs/0.11.3/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/0.11.3/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/index.html
index 07a1bbe2af..fc1407a097 100644
--- a/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/estimators/estimators_vw/index.html b/docs/0.11.3/Quick Examples/estimators/estimators_vw/index.html
index 79cd1d58d8..a22bebcd5b 100644
--- a/docs/0.11.3/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/0.11.3/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/index.html b/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/index.html
index c6b7b01926..2a933f56ce 100644
--- a/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;timestamp&quot;,&quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;timestamp&quot;,&quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1,p._2,1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .++(baseSeq.map(p =&gt; (p._1,p._2,2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;timestamp&quot;,&quot;value&quot;,&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i*imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/transformers/transformers_core/index.html b/docs/0.11.3/Quick Examples/transformers/transformers_core/index.html
index 3a684bd732..b1d9b6d780 100644
--- a/docs/0.11.3/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/0.11.3/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/index.html
index 2ff0c94347..1b88cd7443 100644
--- a/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/transformers/transformers_opencv/index.html b/docs/0.11.3/Quick Examples/transformers/transformers_opencv/index.html
index 9523b07745..d012dacc4b 100644
--- a/docs/0.11.3/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/0.11.3/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Quick Examples/transformers/transformers_vw/index.html b/docs/0.11.3/Quick Examples/transformers/transformers_vw/index.html
index e4d8bec268..b9cef8785c 100644
--- a/docs/0.11.3/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/0.11.3/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/Contributor Guide/index.html b/docs/0.11.3/Reference/Contributor Guide/index.html
index ef34960216..82fb546510 100644
--- a/docs/0.11.3/Reference/Contributor Guide/index.html	
+++ b/docs/0.11.3/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/Developer Setup/index.html b/docs/0.11.3/Reference/Developer Setup/index.html
index a317380780..d03522a046 100644
--- a/docs/0.11.3/Reference/Developer Setup/index.html	
+++ b/docs/0.11.3/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/Docker Setup/index.html b/docs/0.11.3/Reference/Docker Setup/index.html
index c21a202f34..7bd266059e 100644
--- a/docs/0.11.3/Reference/Docker Setup/index.html	
+++ b/docs/0.11.3/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/Dotnet Setup/index.html b/docs/0.11.3/Reference/Dotnet Setup/index.html
index 59b8ceb9f6..f4d6ad52e8 100644
--- a/docs/0.11.3/Reference/Dotnet Setup/index.html	
+++ b/docs/0.11.3/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The cognitive services on spark aint bad|   en-US| null|[{negative, null, {0.0, 0.01, 0.99}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/0.11.3/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/index.html
index fd0ade04bd..9f35e09618 100644
--- a/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/0.11.3/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Reference/R Setup/index.html b/docs/0.11.3/Reference/R Setup/index.html
index d15beea1dd..84aa9f109d 100644
--- a/docs/0.11.3/Reference/R Setup/index.html	
+++ b/docs/0.11.3/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Use with MLFlow/Autologging/index.html b/docs/0.11.3/Use with MLFlow/Autologging/index.html
index 356102e713..7c59e26f28 100644
--- a/docs/0.11.3/Use with MLFlow/Autologging/index.html	
+++ b/docs/0.11.3/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Use with MLFlow/Install/index.html b/docs/0.11.3/Use with MLFlow/Install/index.html
index afead3fbe6..3ff83f5aca 100644
--- a/docs/0.11.3/Use with MLFlow/Install/index.html	
+++ b/docs/0.11.3/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.3/Use with MLFlow/Overview/index.html b/docs/0.11.3/Use with MLFlow/Overview/index.html
index 5cfd6285b4..13b6f2402f 100644
--- a/docs/0.11.3/Use with MLFlow/Overview/index.html	
+++ b/docs/0.11.3/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cognitive-services">Cognitive Services<a href="#cognitive-services" class="hash-link" aria-label="Direct link to Cognitive Services" title="Direct link to Cognitive Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#cognitive-services" class="table-of-contents__link toc-highlight">Cognitive Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Deploy Models/Overview/index.html b/docs/0.11.4/Deploy Models/Overview/index.html
index 1cdd8d0fcf..3f2b3e633f 100644
--- a/docs/0.11.4/Deploy Models/Overview/index.html	
+++ b/docs/0.11.4/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/index.html
index 4cfeb06bbe..b57c5ff027 100644
--- a/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 8870998738..fce1d4fe04 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Cognitive Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(&quot;cognitive-api-key&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the cognitive service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.text import TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The cognitive services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.document.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive.text import TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(&quot;cognitive-api-key-2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/index.html
index 8cc9ed5b89..8a3ebc2d9a 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/en-us/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/en-us/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/en-us/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/en-us/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/big-data/getting-started#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/en-us/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/en-us/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/en-us/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/en-us/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index 132bccd4f2..fdb754b17c 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Cognitive Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure Cognitive Services on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or timeseries, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure Cognitive Services, and we then use to the model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors. </p><p>To learn more about the Anomaly Detector Cognitive Service, refer to <a href="https://docs.microsoft.com/azure/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">item</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> item </span><span class="token keyword">in</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Overview/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Overview/index.html
index a363b28d2d..87347d8677 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Cognitive Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><p><a href="https://azure.microsoft.com/services/cognitive-services/" target="_blank" rel="noopener noreferrer">Azure Cognitive Services</a> are a suite of APIs, SDKs, and services available to help developers build intelligent applications without having direct AI or data science skills or knowledge by enabling developers to easily add cognitive features into their applications. The goal of Azure Cognitive Services is to help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure Cognitive Services can be categorized into five main pillars - Vision, Speech, Language, Web Search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.vision.html#module-synapse.ml.cognitive.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.face.html#module-synapse.ml.cognitive.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.speech.html#module-synapse.ml.cognitive.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/text-analytics/" target="_blank" rel="noopener noreferrer"><strong>Text Analytics</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.text.html#module-synapse.ml.cognitive.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.translate.html#module-synapse.ml.cognitive.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.form.html#module-synapse.ml.cognitive.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.anomaly.html#module-synapse.ml.cognitive.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.anomaly.html#module-synapse.ml.cognitive.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/cognitive-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.bing.html#module-synapse.ml.cognitive.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/scala/com/microsoft/azure/synapse/ml/cognitive/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cognitive.search.html#module-synapse.ml.cognitive.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/cognitive-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Cognitive Services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general Cognitive Services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics-sample">Text Analytics sample<a href="#text-analytics-sample" class="hash-link" aria-label="Direct link to Text Analytics sample" title="Direct link to Text Analytics sample">​</a></h2><p>The <a href="https://azure.microsoft.com/services/cognitive-services/text-analytics/" target="_blank" rel="noopener noreferrer">Text Analytics</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.document.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics-for-health-sample">Text Analytics for Health Sample<a href="#text-analytics-for-health-sample" class="hash-link" aria-label="Direct link to Text Analytics for Health Sample" title="Direct link to Text Analytics for Health Sample">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/cognitive-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured texts such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator-sample">Translator sample<a href="#translator-sample" class="hash-link" aria-label="Direct link to Translator sample" title="Direct link to Translator sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure Cognitive Services family of cognitive APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used for text translation with any operating system. In this sample, we do a simple text translation by providing the sentences you want to translate and target languages you want to translate to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-sample">Form Recognizer sample<a href="#form-recognizer-sample" class="hash-link" aria-label="Direct link to Form Recognizer sample" title="Direct link to Form Recognizer sample">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. Identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more. In this sample, we analyze a business card image and extract its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer">Computer Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions. In this sample, we tag a list of images. Tags are one-word descriptions of things in the image like recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query. In this sample, we use a text query that looks for images with quotes. It returns a list of image URLs that contain photos related to our query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text-sample">Speech-to-Text sample<a href="#speech-to-text-sample" class="hash-link" aria-label="Direct link to Speech-to-Text sample" title="Direct link to Speech-to-Text sample">​</a></h2><p>The <a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. In this sample, we transcribe one audio file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/text-to-speech/#overview" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows one to build apps and services that speak naturally, choosing from more than 270 neural voices across 119 languages and variants.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detector-sample">Anomaly Detector sample<a href="#anomaly-detector-sample" class="hash-link" aria-label="Direct link to Anomaly Detector sample" title="Direct link to Anomaly Detector sample">​</a></h2><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. In this sample, we use the service to find anomalies in the entire time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="arbitrary-web-apis">Arbitrary web APIs<a href="#arbitrary-web-apis" class="hash-link" aria-label="Direct link to Arbitrary web APIs" title="Direct link to Arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#text-analytics-sample" class="table-of-contents__link toc-highlight">Text Analytics sample</a></li><li><a href="#text-analytics-for-health-sample" class="table-of-contents__link toc-highlight">Text Analytics for Health Sample</a></li><li><a href="#translator-sample" class="table-of-contents__link toc-highlight">Translator sample</a></li><li><a href="#form-recognizer-sample" class="table-of-contents__link toc-highlight">Form Recognizer sample</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#speech-to-text-sample" class="table-of-contents__link toc-highlight">Speech-to-Text sample</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#anomaly-detector-sample" class="table-of-contents__link toc-highlight">Anomaly Detector sample</a></li><li><a href="#arbitrary-web-apis" class="table-of-contents__link toc-highlight">Arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index 6010474281..cffb820b32 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Cognitive Services on Spark</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index e62d685902..fffa851a22 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="theme-doc-markdown markdown"><h1>Cognitive Services - Analyze Text</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Creare a Visual Search Engine</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html
index b2d689fb97..780b41ae12 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Creare a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/en-us/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/en-us/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index efcc2b3dbb..6507436834 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your cognitive service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Creare a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index e32ba4d59c..3a8fce5d03 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/cognitive-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/cognitive-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install langchain openai</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index 31e5a0c96a..d98d688ecd 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index d8f3243827..74ba211343 100644
--- a/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Creare a Visual Search Engine/">Quickstart - Creare a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Cognitive Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Cognitive Services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/en-us/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure Cognitive Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account-cli?tabs=linux" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(&quot;anomaly-api-key&quot;)  # Paste your anomaly detector key here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location = &quot;westus2&quot;  # Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals = spark.read.csv(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-cognitive-services-on-spark">Run anomaly detection using Cognitive Services on Spark<a href="#run-anomaly-detection-using-cognitive-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using Cognitive Services on Spark" title="Direct link to Run anomaly detection using Cognitive Services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import col, struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.cognitive import SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.spark import FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(location)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setGroupbyCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSensitivity(95)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setGranularity(&quot;secondly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals.where(col(&quot;unitSymbol&quot;) == &quot;RPM&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;timestamp&quot;, col(&quot;dateTime&quot;).cast(&quot;string&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;value&quot;, col(&quot;measureValue&quot;).cast(&quot;double&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;grouping&quot;, struct(&quot;deviceId&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .mlTransform(detector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly.createOrReplaceTempView(&quot;df_anomaly&quot;)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly.select(&quot;timestamp&quot;, &quot;value&quot;, &quot;deviceId&quot;, &quot;anomalies.isAnomaly&quot;).show(3)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device = spark.sql(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">limit 200&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf = df_anomaly_single_device.toPandas()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset = df_anomaly_single_device.where(col(&quot;isAnomaly&quot;) == 1).toPandas()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.figure(figsize=(23, 8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedUpperValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;darkred&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=0.25,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;UpperMargin&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;darkgreen&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=2,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Expected Value&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;measureValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;b&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;royalblue&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=2,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Actual&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;timestamp&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf[&quot;expectedLowerValue&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color=&quot;black&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth=0.25,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label=&quot;Lower Margin&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.plot(adf_subset[&quot;timestamp&quot;], adf_subset[&quot;measureValue&quot;], &quot;ro&quot;, label=&quot;Anomaly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.legend()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.title(&quot;RPM Anomalies with Confidence Intervals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt.show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure Cognitive Services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-cognitive-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using Cognitive Services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index f7f4470564..5468305ba2 100644
--- a/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/index.html b/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/index.html
index 831a0b941d..16c8d7cea3 100644
--- a/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index f20ebf05ba..0c74e1c66f 100644
--- a/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 0ab6bda187..9bec18aed7 100644
--- a/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index e899dcfc9c..ca4bf3445e 100644
--- a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index ebeeda84af..a51af5a6d9 100644
--- a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 9ecb8d1361..047bcce572 100644
--- a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 7f97e57007..dce785a723 100644
--- a/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 0712e7b769..b9a49661a6 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/index.html
index a12a1334ea..ff49b1febe 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:0.11.4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/index.html
index 183981e665..db541d18bd 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index 4b8ade37c8..17a1fb02c0 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index 7339f2abea..90bc580f97 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index 4cc030e97c..88fffe510a 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index ef445e4d4b..d74d59acb0 100644
--- a/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 0702a5f32e..6d0ccd6565 100644
--- a/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index f737dc4c9d..c53961827d 100644
--- a/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/LightGBM/Overview/index.html b/docs/0.11.4/Explore Algorithms/LightGBM/Overview/index.html
index 949381c27f..67e99a9a74 100644
--- a/docs/0.11.4/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/0.11.4/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index e5442c3d0a..af95043e8a 100644
--- a/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/0.11.4/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/index.html b/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/index.html
index 8f9e59423f..abdb0b6725 100644
--- a/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai langchain pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six pytesseract unstructured</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index 4d70707c99..7b0b777ffd 100644
--- a/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index 07a8f0a560..af306884d4 100644
--- a/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in Cognitive Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/OpenAI/index.html b/docs/0.11.4/Explore Algorithms/OpenAI/index.html
index e2778345da..fd8856a579 100644
--- a/docs/0.11.4/Explore Algorithms/OpenAI/index.html	
+++ b/docs/0.11.4/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/index.html
index e2dd2c885c..fc92d7ad37 100644
--- a/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index fff4b3da03..c21fbd3cae 100644
--- a/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index 70a89111ee..c4018c61e5 100644
--- a/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:0.11.4
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index fc38ab7e46..545c63243d 100644
--- a/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 49b12e7acf..5ebe1eb10b 100644
--- a/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index 75692ad481..88bfa50efd 100644
--- a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index e07318b9b5..9196662053 100644
--- a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 809725cd78..3ffd4e2e07 100644
--- a/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index 42ba57fbfd..d92848b2c3 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index 65469e9aaa..c99fffc3df 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/index.html
index 7b401367d8..b19faf3a19 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index 56932470b8..6935964e59 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index eeb558b471..5c8ef0b934 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 5338645d08..d0182c4ea0 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index f81a10a73d..f36f46c851 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index dd0b8f353c..e9a02d4420 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/index.html
index d1e407b6f9..2065c84a66 100644
--- a/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array = udf(lambda vec: vec.toArray().tolist(), ArrayType(FloatType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vec2array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vec2array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index af8ab76ec4..3fd380c318 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index bb5460e5ea..7e776b2945 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index 26a49a7f56..e4935cd8b6 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index f1676c0584..9531267ae8 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index 26da61adb0..f9f2e2f7c7 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index abb45e405f..cd5c805ee8 100644
--- a/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Get Started/Create a Spark Cluster/index.html b/docs/0.11.4/Get Started/Create a Spark Cluster/index.html
index 10691ae4fc..71da7d4f23 100644
--- a/docs/0.11.4/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/0.11.4/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Get Started/Install SynapseML/index.html b/docs/0.11.4/Get Started/Install SynapseML/index.html
index 120e05c9c8..2045e4f404 100644
--- a/docs/0.11.4/Get Started/Install SynapseML/index.html	
+++ b/docs/0.11.4/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/0.11.4/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Get Started/Quickstart - Your First Models/index.html b/docs/0.11.4/Get Started/Quickstart - Your First Models/index.html
index 947abb2d26..e0f43578f0 100644
--- a/docs/0.11.4/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/0.11.4/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure Cognitive Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-cognitive-services-to-transform-data-in-one-step">Use Cognitive Services to transform data in one step<a href="#use-cognitive-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Cognitive Services to transform data in one step" title="Direct link to Use Cognitive Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Cognitive Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-cognitive-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Cognitive Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Get Started/Set up Cognitive Services/index.html b/docs/0.11.4/Get Started/Set up Cognitive Services/index.html
index b306e22e50..32ef2e0f3b 100644
--- a/docs/0.11.4/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/0.11.4/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Cognitive Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Cognitive Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="cognitive-services">Cognitive Services<a href="#cognitive-services" class="hash-link" aria-label="Direct link to Cognitive Services" title="Direct link to Cognitive Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/products/cognitive-services/" target="_blank" rel="noopener noreferrer">Cognitive Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Cognitive Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create an Azure Cognitive multi-service (Decision, Language, Speech, Vision) resource</a>. Alternatively, you can follow the steps to <a href="https://learn.microsoft.com/en-us/azure/cognitive-services/cognitive-services-apis-create-account?tabs=decision%2Canomaly-detector%2Clanguage-service%2Ccomputer-vision%2Cwindows#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">create Single-service resource</a>. </li><li>Get your Cognitive Service resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#cognitive-services" class="table-of-contents__link toc-highlight">Cognitive Services</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Overview/index.html b/docs/0.11.4/Overview/index.html
index 16be19a797..a201dff6d2 100644
--- a/docs/0.11.4/Overview/index.html
+++ b/docs/0.11.4/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/estimators/estimators_causal/index.html b/docs/0.11.4/Quick Examples/estimators/estimators_causal/index.html
index ab269200ca..daf348ab8e 100644
--- a/docs/0.11.4/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/0.11.4/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/index.html b/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/index.html
index ae4070c5ec..3b10c6742a 100644
--- a/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/estimators/estimators_core/index.html b/docs/0.11.4/Quick Examples/estimators/estimators_core/index.html
index 6e28ac478f..1a63facb22 100644
--- a/docs/0.11.4/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/0.11.4/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/index.html
index e1e626226c..7e8718d186 100644
--- a/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/estimators/estimators_vw/index.html b/docs/0.11.4/Quick Examples/estimators/estimators_vw/index.html
index 5d6df43371..28693fe8fd 100644
--- a/docs/0.11.4/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/0.11.4/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/index.html b/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/index.html
index 98bfd3289d..495eb07783 100644
--- a/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                              .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;timestamp&quot;,&quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;timestamp&quot;,&quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1,p._2,1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .++(baseSeq.map(p =&gt; (p._1,p._2,2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;timestamp&quot;,&quot;value&quot;,&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.cognitive.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i*imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/transformers/transformers_core/index.html b/docs/0.11.4/Quick Examples/transformers/transformers_core/index.html
index 45b4ab82b7..6e7155c9bc 100644
--- a/docs/0.11.4/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/0.11.4/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/index.html
index fdf5a2e7d1..c955c5c73e 100644
--- a/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/transformers/transformers_opencv/index.html b/docs/0.11.4/Quick Examples/transformers/transformers_opencv/index.html
index 15cdaea8ce..8f958ec64a 100644
--- a/docs/0.11.4/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/0.11.4/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Quick Examples/transformers/transformers_vw/index.html b/docs/0.11.4/Quick Examples/transformers/transformers_vw/index.html
index 15c8c52b0b..594b53939c 100644
--- a/docs/0.11.4/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/0.11.4/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/Contributor Guide/index.html b/docs/0.11.4/Reference/Contributor Guide/index.html
index 7f9fe2606a..2e67b4b5ea 100644
--- a/docs/0.11.4/Reference/Contributor Guide/index.html	
+++ b/docs/0.11.4/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/Developer Setup/index.html b/docs/0.11.4/Reference/Developer Setup/index.html
index a6210240f7..57ad8d2ad6 100644
--- a/docs/0.11.4/Reference/Developer Setup/index.html	
+++ b/docs/0.11.4/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/Docker Setup/index.html b/docs/0.11.4/Reference/Docker Setup/index.html
index af97cabc01..f8d74f82d6 100644
--- a/docs/0.11.4/Reference/Docker Setup/index.html	
+++ b/docs/0.11.4/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/Dotnet Setup/index.html b/docs/0.11.4/Reference/Dotnet Setup/index.html
index 84d902c737..2a04015eb6 100644
--- a/docs/0.11.4/Reference/Dotnet Setup/index.html	
+++ b/docs/0.11.4/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The cognitive services on spark aint bad|   en-US| null|[{negative, null, {0.0, 0.01, 0.99}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/0.11.4/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/index.html
index 2dd0f2e1aa..ae1992df4d 100644
--- a/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/0.11.4/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Reference/R Setup/index.html b/docs/0.11.4/Reference/R Setup/index.html
index 0d127b6b81..641e53488d 100644
--- a/docs/0.11.4/Reference/R Setup/index.html	
+++ b/docs/0.11.4/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Use with MLFlow/Autologging/index.html b/docs/0.11.4/Use with MLFlow/Autologging/index.html
index 0ac7f7de38..b76ad697de 100644
--- a/docs/0.11.4/Use with MLFlow/Autologging/index.html	
+++ b/docs/0.11.4/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Use with MLFlow/Install/index.html b/docs/0.11.4/Use with MLFlow/Install/index.html
index c449edb962..0ad6b70532 100644
--- a/docs/0.11.4/Use with MLFlow/Install/index.html	
+++ b/docs/0.11.4/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/0.11.4/Use with MLFlow/Overview/index.html b/docs/0.11.4/Use with MLFlow/Overview/index.html
index 44da29248b..e437e6f786 100644
--- a/docs/0.11.4/Use with MLFlow/Overview/index.html	
+++ b/docs/0.11.4/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/0.11.4/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/0.11.4/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>0.11.4</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 0.11.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cognitive-services">Cognitive Services<a href="#cognitive-services" class="hash-link" aria-label="Direct link to Cognitive Services" title="Direct link to Cognitive Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cognitive </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#cognitive-services" class="table-of-contents__link toc-highlight">Cognitive Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Deploy Models/Overview/index.html b/docs/1.0.1/Deploy Models/Overview/index.html
index 317e5f7eaa..15eb3847d0 100644
--- a/docs/1.0.1/Deploy Models/Overview/index.html	
+++ b/docs/1.0.1/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/index.html
index 323fe774d9..a0159822a4 100644
--- a/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index e11e051d3e..3f435d7fa9 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>AI Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the AI service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.language import AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The AI services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKind(&quot;SentimentAnalysis&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.documents.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key-2&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/index.html
index 6039460cad..53e24e705d 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/overview-cognitive-services#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index d25ff19c16..0cb0fe1b2b 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: AI Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure AI services on Apache Spark to detect anomalies in multivariate time-series data. Multivariate anomaly detection takes correlations and dependencies between the different variables into account when discovering anomalies. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI Services, and we then use to the model to detect anomalies in a  dataset of synthetic measurements from three IoT sensors. </p><p>To learn more about the Azure AI Anomaly Detector, refer to <a href="https://docs.microsoft.com/azure/ai-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><p>For Spark3.3 and below versions, the output of select statements will be in the format of <code>List&lt;Rows&gt;</code>, so to format the data into dictionary and generate the values when interpretation is empty, please use the below parse method:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def parse(x):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if len(x) &gt; 0:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return dict([item[:2] for item in x])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return {&quot;sensor_1&quot;: 0, &quot;sensor_2&quot;: 0, &quot;sensor_3&quot;: 0}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Staring with Spark3.4, the output of the select statement is already formatted as a <code>numpy.ndarry&lt;dictionary&gt;</code> and no need to format the data again, so please use below parse method to generate the values when interpretation is empty:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Overview/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Overview/index.html
index 9bd7e1f782..1743325f95 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure AI Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><p><a href="https://azure.microsoft.com/services/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI services</a> is a suite of APIs, SDKs, and services that developers can use to add intelligent features to their applications. AI services empower developers even when they don&#x27;t have direct AI or data science skills or knowledge. Azure AI services help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure AI services can be categorized into five main pillars: Vision, Speech, Language, Web search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-language" target="_blank" rel="noopener noreferrer"><strong>AI Language</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/services-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.bing.html#module-synapse.ml.services.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/services-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Azure AI services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-your-system">Prepare your system<a href="#prepare-your-system" class="hash-link" aria-label="Direct link to Prepare your system" title="Direct link to Prepare your system">​</a></h2><p>To begin, import required libraries and initialize your Spark session.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Import Azure AI services libraries and replace the keys in the following code snippet with your Azure AI services key.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general AI services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-sentiment-analysis-on-text">Perform sentiment analysis on text<a href="#perform-sentiment-analysis-on-text" class="hash-link" aria-label="Direct link to Perform sentiment analysis on text" title="Direct link to Perform sentiment analysis on text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-language/" target="_blank" rel="noopener noreferrer">AI Language</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The AI services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-text-analytics-for-health-data">Perform text analytics for health data<a href="#perform-text-analytics-for-health-data" class="hash-link" aria-label="Direct link to Perform text analytics for health data" title="Direct link to Perform text analytics for health data">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/ai-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured text such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><p>The following code sample analyzes and transforms text from doctors notes into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translate-text-into-a-different-language">Translate text into a different language<a href="#translate-text-into-a-different-language" class="hash-link" aria-label="Direct link to Translate text into a different language" title="Direct link to Translate text into a different language">​</a></h2><p><a href="https://azure.microsoft.com/services/ai-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure AI services family of AI APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used to translate text without hosting your own algorithm.</p><p>The following code sample does a simple text translation by providing the sentences you want to translate and target languages you want to translate them to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="extract-information-from-a-document-into-structured-data">Extract information from a document into structured data<a href="#extract-information-from-a-document-into-structured-data" class="hash-link" aria-label="Direct link to Extract information from a document into structured data" title="Direct link to Extract information from a document into structured data">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. With Form Recognizer, you can identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more.</p><p>The following code sample analyzes a business card image and extracts its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-vision/" target="_blank" rel="noopener noreferrer">Azure AI Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions.</p><p>The following code sample analyzes images and labels them with tags. Tags are one-word descriptions of things in the image, such as recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://www.microsoft.com/bing/apis/bing-image-search-api" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query.</p><p>The following code sample uses a text query that looks for images with quotes. The output of the code is a list of image URLs that contain photos related to the query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-speech-to-text">Transform speech to text<a href="#transform-speech-to-text" class="hash-link" aria-label="Direct link to Transform speech to text" title="Direct link to Transform speech to text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-speech/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. The following code sample transcribes one audio file to text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/text-to-speech/" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows you to build applications that speak naturally. You can choose from more than 270 neural voices across 119 languages.</p><p>The following code sample transforms text into an audio file that contains the content of the text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="detect-anomalies-in-time-series-data">Detect anomalies in time series data<a href="#detect-anomalies-in-time-series-data" class="hash-link" aria-label="Direct link to Detect anomalies in time series data" title="Direct link to Detect anomalies in time series data">​</a></h2><p>If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one. You may want to skip this part.</p><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. The following code sample uses the Anomaly Detector service to find anomalies in a time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-information-from-arbitrary-web-apis">Get information from arbitrary web APIs<a href="#get-information-from-arbitrary-web-apis" class="hash-link" aria-label="Direct link to Get information from arbitrary web APIs" title="Direct link to Get information from arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#prepare-your-system" class="table-of-contents__link toc-highlight">Prepare your system</a></li><li><a href="#perform-sentiment-analysis-on-text" class="table-of-contents__link toc-highlight">Perform sentiment analysis on text</a></li><li><a href="#perform-text-analytics-for-health-data" class="table-of-contents__link toc-highlight">Perform text analytics for health data</a></li><li><a href="#translate-text-into-a-different-language" class="table-of-contents__link toc-highlight">Translate text into a different language</a></li><li><a href="#extract-information-from-a-document-into-structured-data" class="table-of-contents__link toc-highlight">Extract information from a document into structured data</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#transform-speech-to-text" class="table-of-contents__link toc-highlight">Transform speech to text</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#detect-anomalies-in-time-series-data" class="table-of-contents__link toc-highlight">Detect anomalies in time series data</a></li><li><a href="#get-information-from-arbitrary-web-apis" class="table-of-contents__link toc-highlight">Get information from arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index e1620a1405..fa2d5b83bd 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Azure AI Services</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index fe2e6b38a9..23317f92b5 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Analyze Text with SynapseML and Azure AI Language</h1><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/overview" target="_blank" rel="noopener noreferrer">Azure AI Language</a> is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.
 You can use SynapseML with Azure AI Language for <strong>named entity recognition</strong>, <strong>language detection</strong>, <strong>entity linking</strong>, <strong>key phrase extraction</strong>, <strong>Pii entity recognition</strong> and <strong>sentiment analysis</strong>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="named-entity-recognition">Named Entity Recognition<a href="#named-entity-recognition" class="hash-link" aria-label="Direct link to Named Entity Recognition" title="Direct link to Named Entity Recognition">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/overview" target="_blank" rel="noopener noreferrer">Named Entity Recognition</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. The NER feature can identify and categorize entities in unstructured text. For example: people, places, organizations, and quantities. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/language-support?tabs=ga-api" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">language</th><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">en</td><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td align="left">en</td><td align="left">I had a wonderful trip to Seattle last week.</td><td align="left">[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetection">LanguageDetection<a href="#languagedetection" class="hash-link" aria-label="Direct link to LanguageDetection" title="Direct link to LanguageDetection">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/overview" target="_blank" rel="noopener noreferrer">Language detection</a> can detect the language a document is written in. It returns a language code for a wide range of languages, variants, dialects, and some regional/cultural languages. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a document written in English.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;这是一份用中文写的文件&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language_detection </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LanguageDetection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> language_detection</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language.documents.detectedLanguage&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">detectedLanguage</th></tr></thead><tbody><tr><td align="left">This is a document written in English.</td><td align="left">{&quot;name&quot;: &quot;English&quot;, &quot;iso6391Name&quot;: &quot;en&quot;, &quot;confidenceScore&quot;: 0.99}</td></tr><tr><td align="left">这是一份用中文写的文件</td><td align="left">{&quot;name&quot;: &quot;Chinese_Simplified&quot;, &quot;iso6391Name&quot;: &quot;zh_chs&quot;, &quot;confidenceScore&quot;: 1}</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entitylinking">EntityLinking<a href="#entitylinking" class="hash-link" aria-label="Direct link to EntityLinking" title="Direct link to EntityLinking">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/overview" target="_blank" rel="noopener noreferrer">Entity linking</a> identifies and disambiguates the identity of entities found in text. For example, in the sentence &quot;We went to Seattle last week.&quot;, the word &quot;Seattle&quot; would be identified, with a link to more information on Wikipedia. <a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/language-support" target="_blank" rel="noopener noreferrer">English and Spanish are supported</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;We went to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_linking </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityLinking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_linking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[{&quot;bingId&quot;: &quot;a093e9b9-90f5-a3d5-c4b8-5855e1b01f85&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Microsoft&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.48, &quot;length&quot;: 9, &quot;offset&quot;: 0, &quot;text&quot;: &quot;Microsoft&quot;}]<!-- -->, &quot;name&quot;: &quot;Microsoft&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Microsoft%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Microsoft&quot;}</a>, {&quot;bingId&quot;: &quot;0d47c987-0042-5576-15e8-97af601614fa&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Bill Gates&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.52, &quot;length&quot;: 10, &quot;offset&quot;: 25, &quot;text&quot;: &quot;Bill Gates&quot;}]<!-- -->, &quot;name&quot;: &quot;Bill Gates&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Bill_Gates%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Bill_Gates&quot;}</a>, {&quot;bingId&quot;: &quot;df2c4376-9923-6a54-893f-2ee5a5badbc7&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Paul Allen&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.54, &quot;length&quot;: 10, &quot;offset&quot;: 40, &quot;text&quot;: &quot;Paul Allen&quot;}]<!-- -->, &quot;name&quot;: &quot;Paul Allen&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Paul_Allen%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Paul_Allen&quot;}</a>, {&quot;bingId&quot;: &quot;52535f87-235e-b513-54fe-c03e4233ac6e&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;April 4&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.38, &quot;length&quot;: 7, &quot;offset&quot;: 54, &quot;text&quot;: &quot;April 4&quot;}]<!-- -->, &quot;name&quot;: &quot;April 4&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/April_4%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/April_4&quot;}]</a></td></tr><tr><td align="left">We went to Seattle last week.</td><td align="left">[{&quot;bingId&quot;: &quot;5fbba6b8-85e1-4d41-9444-d9055436e473&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Seattle&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.17, &quot;length&quot;: 7, &quot;offset&quot;: 11, &quot;text&quot;: &quot;Seattle&quot;}]<!-- -->, &quot;name&quot;: &quot;Seattle&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Seattle%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Seattle&quot;}]</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextraction">KeyPhraseExtraction<a href="#keyphraseextraction" class="hash-link" aria-label="Direct link to KeyPhraseExtraction" title="Direct link to KeyPhraseExtraction">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/overview" target="_blank" rel="noopener noreferrer">Key phrase extraction</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. Use key phrase extraction to quickly identify the main concepts in text. For example, in the text &quot;The food was delicious and the staff were wonderful.&quot;, key phrase extraction will return the main topics: &quot;food&quot; and &quot;wonderful staff&quot;. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/key-phrase-extraction/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key_phrase_extraction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;KeyPhraseExtraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> key_phrase_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction.documents.keyPhrases&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">keyPhrases</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[&quot;Bill Gates&quot;, &quot;Paul Allen&quot;, &quot;Microsoft&quot;, &quot;April&quot;]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[&quot;modern medical office&quot;, &quot;Dr. Smith&quot;, &quot;great staff&quot;]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="piientityrecognition">PiiEntityRecognition<a href="#piientityrecognition" class="hash-link" aria-label="Direct link to PiiEntityRecognition" title="Direct link to PiiEntityRecognition">​</a></h2><p>The PII detection feature can identify, categorize, and redact sensitive information in unstructured text. For example: phone numbers, email addresses, and forms of identification. The method for utilizing PII in conversations is different than other use cases, and articles for this use have been separated. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/language-support?tabs=documents" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Call our office at 312-555-1234, or send an email to support@contoso.com&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii_entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PiiEntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> pii_entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Call our office at 312-555-1234, or send an email to <a href="mailto:support@contoso.com" target="_blank" rel="noopener noreferrer">support@contoso.com</a></td><td align="left">[{&quot;category&quot;: &quot;PhoneNumber&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 12, &quot;offset&quot;: 19, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;312-555-1234&quot;}, {&quot;category&quot;: &quot;Email&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 19, &quot;offset&quot;: 53, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;support@contoso.com&quot;}]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.93, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sentimentanalysis">SentimentAnalysis<a href="#sentimentanalysis" class="hash-link" aria-label="Direct link to SentimentAnalysis" title="Direct link to SentimentAnalysis">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview" target="_blank" rel="noopener noreferrer">Sentiment analysis</a> and opinion mining are features offered by the Language service, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. These features help you find out what people think of your brand or topic by mining text for clues about positive or negative sentiment, and can associate them with specific aspects of the text. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/sentiment-opinion-mining/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The food and service were unacceptable. The concierge was nice, however.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;It taste great.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment_analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> sentiment_analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">sentiment</th></tr></thead><tbody><tr><td align="left">The food and service were unacceptable. The concierge was nice, however.</td><td align="left">mixed</td></tr><tr><td align="left">It tastes great.</td><td align="left">positive</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="analyze-text-with-textanalyze">Analyze Text with TextAnalyze<a href="#analyze-text-with-textanalyze" class="hash-link" aria-label="Direct link to Analyze Text with TextAnalyze" title="Direct link to Analyze Text with TextAnalyze">​</a></h2><p>Text Analyze is Deprecated, please use AnalyzeText instead</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#named-entity-recognition" class="table-of-contents__link toc-highlight">Named Entity Recognition</a></li><li><a href="#languagedetection" class="table-of-contents__link toc-highlight">LanguageDetection</a></li><li><a href="#entitylinking" class="table-of-contents__link toc-highlight">EntityLinking</a></li><li><a href="#keyphraseextraction" class="table-of-contents__link toc-highlight">KeyPhraseExtraction</a></li><li><a href="#piientityrecognition" class="table-of-contents__link toc-highlight">PiiEntityRecognition</a></li><li><a href="#sentimentanalysis" class="table-of-contents__link toc-highlight">SentimentAnalysis</a></li><li><a href="#analyze-text-with-textanalyze" class="table-of-contents__link toc-highlight">Analyze Text with TextAnalyze</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index 7493403e64..a75770d69b 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your Azure AI service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html
index b13528008f..8a12b9eb34 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index 55208c6f2a..02e6030420 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index 132b59244f..c49b699dc8 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index 386be9f704..de4333ac5b 100644
--- a/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Azure AI Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure AI Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://learn.microsoft.com/azure/ai-services/multi-service-resource" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector key here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-ai-services-on-spark">Run anomaly detection using AI services on Spark<a href="#run-anomaly-detection-using-ai-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using AI services on Spark" title="Direct link to Run anomaly detection using AI services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitivity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;secondly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unitSymbol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dateTime&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createOrReplaceTempView</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;df_anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">limit 200&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedUpperValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;UpperMargin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Expected Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;royalblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedLowerValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Lower Margin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ro&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM Anomalies with Confidence Intervals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure AI services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-ai-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using AI services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 41ed040f9c..1890a8d2b0 100644
--- a/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/index.html b/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/index.html
index 2500794e7e..4088ea32bb 100644
--- a/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 9dad37952d..edb693e5b2 100644
--- a/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 363db07b38..0d94bfdb57 100644
--- a/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index a9a6001863..b9a59c28e4 100644
--- a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index 131ba5a1aa..0423866917 100644
--- a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 8e6c324fe9..9057a754e2 100644
--- a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 243e5d6e90..2c72f7ad27 100644
--- a/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 75031e7c61..f0d42c4c9f 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/index.html
index 19394bfdd1..fcf059f98e 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:1.0.1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/index.html
index 9d6e51b102..47f20fd591 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index 38a836a29e..a6db548dd8 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index 4a2bc208af..d9116c1040 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index 71ec74c053..af3453f97b 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm onnxmltools</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.7</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index cb27d90235..5ea27b7453 100644
--- a/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 16c4e56c65..583f8579cf 100644
--- a/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index 97d22d034d..905b31898f 100644
--- a/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/LightGBM/Overview/index.html b/docs/1.0.1/Explore Algorithms/LightGBM/Overview/index.html
index d2bb592fc3..268ea6f694 100644
--- a/docs/1.0.1/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/1.0.1/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index 1a031fa6ff..2e7b25472a 100644
--- a/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/1.0.1/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/index.html b/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/index.html
index 01ba364b23..d7eddfbed2 100644
--- a/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook into <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a>, <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">Synapse Workspace</a> or if using Databricks into the <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">Databricks Workspace</a>.</li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported.</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><span class="token plain"> pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six unstructured</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.10</span><span class="token number" style="color:rgb(174, 129, 255)">.24</span><span class="token plain"> pytesseract numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><p>Note that you also need to set up your Bing search to gain access to your <a href="https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource" target="_blank" rel="noopener noreferrer">Bing Search subscription key</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index 821f064d23..01e3b0781f 100644
--- a/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index 9ae499d92d..e44521ea67 100644
--- a/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in AI Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/OpenAI/index.html b/docs/1.0.1/Explore Algorithms/OpenAI/index.html
index f6e0d00606..88382cfcd4 100644
--- a/docs/1.0.1/Explore Algorithms/OpenAI/index.html	
+++ b/docs/1.0.1/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/index.html
index 6a1044c42a..3c6956ce15 100644
--- a/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index d9948ddd53..af63565a3b 100644
--- a/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index bed5051a3b..c145fa254b 100644
--- a/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:1.0.1
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 86bfe1cb2c..a56408b3ba 100644
--- a/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 54d011edf8..5ef632bc53 100644
--- a/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index d4f0a18823..8161a98a7a 100644
--- a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index 9331a989d6..abf72a72aa 100644
--- a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 3fc60c7556..a035f02cc7 100644
--- a/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index c2e32aa8a8..5d34d1fc1f 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index 1db80aa434..b7933499b0 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/index.html
index d6e31191e8..84c442d999 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index 243302e38e..a1d0dbef32 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index e5876e9fff..296d945530 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 8564d64341..150602588d 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index 4607b459be..156cd883a5 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index 6e0e3d9267..8498d9ff20 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 9df64192ff..1458d389d0 100644
--- a/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.functions import vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vector_to_array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vector_to_array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index c518308e64..0c1e6fd410 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index cc1e8215ca..2ef31680ad 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index 75939867c6..e70adf195f 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index 5d4943789d..31673ce873 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index 518583e694..408ec1ed03 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 7e261c6c59..2e0c8cbad7 100644
--- a/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Get Started/Create a Spark Cluster/index.html b/docs/1.0.1/Get Started/Create a Spark Cluster/index.html
index 6f8cf4c9a4..bcc39e1fd3 100644
--- a/docs/1.0.1/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/1.0.1/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Get Started/Install SynapseML/index.html b/docs/1.0.1/Get Started/Install SynapseML/index.html
index dcc7d8b2db..231a19112e 100644
--- a/docs/1.0.1/Get Started/Install SynapseML/index.html	
+++ b/docs/1.0.1/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/1.0.1/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Get Started/Quickstart - Your First Models/index.html b/docs/1.0.1/Get Started/Quickstart - Your First Models/index.html
index 2551c888ec..082d3a5abc 100644
--- a/docs/1.0.1/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/1.0.1/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-azure-ai-services-to-transform-data-in-one-step">Use Azure AI Services to transform data in one step<a href="#use-azure-ai-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Azure AI Services to transform data in one step" title="Direct link to Use Azure AI Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Azure AI Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-azure-ai-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Azure AI Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Get Started/Set up Cognitive Services/index.html b/docs/1.0.1/Get Started/Set up Cognitive Services/index.html
index 09f7c76562..e5639d57f2 100644
--- a/docs/1.0.1/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/1.0.1/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Azure AI Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/en-us/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Azure AI Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create Azure AI service (Decision, Language, Speech, Vision) resource</a>. You can follow the steps at <a href="https://learn.microsoft.com/en-us/azure/ai-services/multi-service-resource?tabs=windows&amp;pivots=azportal#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">Create a multi-service resource for Azure AI services</a>. </li><li>Get your Azure AI Services resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Overview/index.html b/docs/1.0.1/Overview/index.html
index 402606d7f5..e4b09582dc 100644
--- a/docs/1.0.1/Overview/index.html
+++ b/docs/1.0.1/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/estimators/estimators_causal/index.html b/docs/1.0.1/Quick Examples/estimators/estimators_causal/index.html
index c20e8703bf..d409b2ba33 100644
--- a/docs/1.0.1/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/1.0.1/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/index.html b/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/index.html
index d35ca9c25d..603321c340 100644
--- a/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/estimators/estimators_core/index.html b/docs/1.0.1/Quick Examples/estimators/estimators_core/index.html
index ac4b8b1cef..0fc1269ec5 100644
--- a/docs/1.0.1/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/1.0.1/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/index.html
index e3011ce12f..eb8b688fc9 100644
--- a/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/estimators/estimators_vw/index.html b/docs/1.0.1/Quick Examples/estimators/estimators_vw/index.html
index 55236f1f22..649094b716 100644
--- a/docs/1.0.1/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/1.0.1/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/index.html b/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/index.html
index fe6298775f..64913d1e3b 100644
--- a/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1, p._2, 1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .++(baseSeq.map(p =&gt; (p._1, p._2, 2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;timestamp&quot;, &quot;value&quot;, &quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i * imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/transformers/transformers_core/index.html b/docs/1.0.1/Quick Examples/transformers/transformers_core/index.html
index c1ad46a22b..c1cd122a64 100644
--- a/docs/1.0.1/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/1.0.1/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/index.html
index ef1a755f26..3d22b28ab0 100644
--- a/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/transformers/transformers_opencv/index.html b/docs/1.0.1/Quick Examples/transformers/transformers_opencv/index.html
index 967223482d..28b0ec33ad 100644
--- a/docs/1.0.1/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/1.0.1/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Quick Examples/transformers/transformers_vw/index.html b/docs/1.0.1/Quick Examples/transformers/transformers_vw/index.html
index b875a06617..230a2423ea 100644
--- a/docs/1.0.1/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/1.0.1/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/Contributor Guide/index.html b/docs/1.0.1/Reference/Contributor Guide/index.html
index 1f71fe933d..f916724631 100644
--- a/docs/1.0.1/Reference/Contributor Guide/index.html	
+++ b/docs/1.0.1/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/Developer Setup/index.html b/docs/1.0.1/Reference/Developer Setup/index.html
index 1833207efb..ab6999925b 100644
--- a/docs/1.0.1/Reference/Developer Setup/index.html	
+++ b/docs/1.0.1/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/Docker Setup/index.html b/docs/1.0.1/Reference/Docker Setup/index.html
index 164d215760..5e9e0fa32e 100644
--- a/docs/1.0.1/Reference/Docker Setup/index.html	
+++ b/docs/1.0.1/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/Dotnet Setup/index.html b/docs/1.0.1/Reference/Dotnet Setup/index.html
index 1030cc992c..8750417b28 100644
--- a/docs/1.0.1/Reference/Dotnet Setup/index.html	
+++ b/docs/1.0.1/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The Azure AI services on spark aint bad|   en-US| null|[{positive, null, {0.99, 0.01, 0.00}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.1/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/index.html
index 06d29f18fa..8a80bfbf96 100644
--- a/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/1.0.1/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Reference/R Setup/index.html b/docs/1.0.1/Reference/R Setup/index.html
index c87ff44e3d..9ac745f4d2 100644
--- a/docs/1.0.1/Reference/R Setup/index.html	
+++ b/docs/1.0.1/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Use with MLFlow/Autologging/index.html b/docs/1.0.1/Use with MLFlow/Autologging/index.html
index 178c4c1acb..b2a74e818a 100644
--- a/docs/1.0.1/Use with MLFlow/Autologging/index.html	
+++ b/docs/1.0.1/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Use with MLFlow/Install/index.html b/docs/1.0.1/Use with MLFlow/Install/index.html
index 6a292c3942..64d2ffd8f5 100644
--- a/docs/1.0.1/Use with MLFlow/Install/index.html	
+++ b/docs/1.0.1/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.1/Use with MLFlow/Overview/index.html b/docs/1.0.1/Use with MLFlow/Overview/index.html
index 6aebae07ff..7f447dcc7c 100644
--- a/docs/1.0.1/Use with MLFlow/Overview/index.html	
+++ b/docs/1.0.1/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.1/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.1/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.1</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.1</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Deploy Models/Overview/index.html b/docs/1.0.2/Deploy Models/Overview/index.html
index 3442d650ad..158162ff0e 100644
--- a/docs/1.0.2/Deploy Models/Overview/index.html	
+++ b/docs/1.0.2/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/index.html
index 908b36c43b..59f2db3095 100644
--- a/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 696c95e530..51421ee810 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>AI Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the AI service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.language import AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The AI services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKind(&quot;SentimentAnalysis&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.documents.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key-2&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/index.html
index dc7a5d5ce3..37fc60fbcc 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/overview-cognitive-services#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index e187284e9f..78cd390f0f 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: AI Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure AI services on Apache Spark to detect anomalies in multivariate time-series data. Multivariate anomaly detection takes correlations and dependencies between the different variables into account when discovering anomalies. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI Services, and we then use to the model to detect anomalies in a  dataset of synthetic measurements from three IoT sensors. </p><p>To learn more about the Azure AI Anomaly Detector, refer to <a href="https://docs.microsoft.com/azure/ai-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><p>For Spark3.3 and below versions, the output of select statements will be in the format of <code>List&lt;Rows&gt;</code>, so to format the data into dictionary and generate the values when interpretation is empty, please use the below parse method:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def parse(x):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if len(x) &gt; 0:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return dict([item[:2] for item in x])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return {&quot;sensor_1&quot;: 0, &quot;sensor_2&quot;: 0, &quot;sensor_3&quot;: 0}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Staring with Spark3.4, the output of the select statement is already formatted as a <code>numpy.ndarry&lt;dictionary&gt;</code> and no need to format the data again, so please use below parse method to generate the values when interpretation is empty:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Overview/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Overview/index.html
index d26c6fae8d..5738e9aeb8 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure AI Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><p><a href="https://azure.microsoft.com/services/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI services</a> is a suite of APIs, SDKs, and services that developers can use to add intelligent features to their applications. AI services empower developers even when they don&#x27;t have direct AI or data science skills or knowledge. Azure AI services help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure AI services can be categorized into five main pillars: Vision, Speech, Language, Web search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-language" target="_blank" rel="noopener noreferrer"><strong>AI Language</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/services-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.bing.html#module-synapse.ml.services.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/services-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Azure AI services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-your-system">Prepare your system<a href="#prepare-your-system" class="hash-link" aria-label="Direct link to Prepare your system" title="Direct link to Prepare your system">​</a></h2><p>To begin, import required libraries and initialize your Spark session.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Import Azure AI services libraries and replace the keys in the following code snippet with your Azure AI services key.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general AI services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-sentiment-analysis-on-text">Perform sentiment analysis on text<a href="#perform-sentiment-analysis-on-text" class="hash-link" aria-label="Direct link to Perform sentiment analysis on text" title="Direct link to Perform sentiment analysis on text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-language/" target="_blank" rel="noopener noreferrer">AI Language</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The AI services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-text-analytics-for-health-data">Perform text analytics for health data<a href="#perform-text-analytics-for-health-data" class="hash-link" aria-label="Direct link to Perform text analytics for health data" title="Direct link to Perform text analytics for health data">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/ai-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured text such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><p>The following code sample analyzes and transforms text from doctors notes into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translate-text-into-a-different-language">Translate text into a different language<a href="#translate-text-into-a-different-language" class="hash-link" aria-label="Direct link to Translate text into a different language" title="Direct link to Translate text into a different language">​</a></h2><p><a href="https://azure.microsoft.com/services/ai-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure AI services family of AI APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used to translate text without hosting your own algorithm.</p><p>The following code sample does a simple text translation by providing the sentences you want to translate and target languages you want to translate them to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="extract-information-from-a-document-into-structured-data">Extract information from a document into structured data<a href="#extract-information-from-a-document-into-structured-data" class="hash-link" aria-label="Direct link to Extract information from a document into structured data" title="Direct link to Extract information from a document into structured data">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. With Form Recognizer, you can identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more.</p><p>The following code sample analyzes a business card image and extracts its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-vision/" target="_blank" rel="noopener noreferrer">Azure AI Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions.</p><p>The following code sample analyzes images and labels them with tags. Tags are one-word descriptions of things in the image, such as recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://www.microsoft.com/bing/apis/bing-image-search-api" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query.</p><p>The following code sample uses a text query that looks for images with quotes. The output of the code is a list of image URLs that contain photos related to the query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-speech-to-text">Transform speech to text<a href="#transform-speech-to-text" class="hash-link" aria-label="Direct link to Transform speech to text" title="Direct link to Transform speech to text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-speech/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. The following code sample transcribes one audio file to text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/text-to-speech/" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows you to build applications that speak naturally. You can choose from more than 270 neural voices across 119 languages.</p><p>The following code sample transforms text into an audio file that contains the content of the text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="detect-anomalies-in-time-series-data">Detect anomalies in time series data<a href="#detect-anomalies-in-time-series-data" class="hash-link" aria-label="Direct link to Detect anomalies in time series data" title="Direct link to Detect anomalies in time series data">​</a></h2><p>If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one. You may want to skip this part.</p><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. The following code sample uses the Anomaly Detector service to find anomalies in a time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-information-from-arbitrary-web-apis">Get information from arbitrary web APIs<a href="#get-information-from-arbitrary-web-apis" class="hash-link" aria-label="Direct link to Get information from arbitrary web APIs" title="Direct link to Get information from arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#prepare-your-system" class="table-of-contents__link toc-highlight">Prepare your system</a></li><li><a href="#perform-sentiment-analysis-on-text" class="table-of-contents__link toc-highlight">Perform sentiment analysis on text</a></li><li><a href="#perform-text-analytics-for-health-data" class="table-of-contents__link toc-highlight">Perform text analytics for health data</a></li><li><a href="#translate-text-into-a-different-language" class="table-of-contents__link toc-highlight">Translate text into a different language</a></li><li><a href="#extract-information-from-a-document-into-structured-data" class="table-of-contents__link toc-highlight">Extract information from a document into structured data</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#transform-speech-to-text" class="table-of-contents__link toc-highlight">Transform speech to text</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#detect-anomalies-in-time-series-data" class="table-of-contents__link toc-highlight">Detect anomalies in time series data</a></li><li><a href="#get-information-from-arbitrary-web-apis" class="table-of-contents__link toc-highlight">Get information from arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index 93ab8b0b68..2e3460d668 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Azure AI Services</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index 3bf8b9d09a..a0f5f35e03 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Analyze Text with SynapseML and Azure AI Language</h1><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/overview" target="_blank" rel="noopener noreferrer">Azure AI Language</a> is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.
 You can use SynapseML with Azure AI Language for <strong>named entity recognition</strong>, <strong>language detection</strong>, <strong>entity linking</strong>, <strong>key phrase extraction</strong>, <strong>Pii entity recognition</strong> and <strong>sentiment analysis</strong>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="named-entity-recognition">Named Entity Recognition<a href="#named-entity-recognition" class="hash-link" aria-label="Direct link to Named Entity Recognition" title="Direct link to Named Entity Recognition">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/overview" target="_blank" rel="noopener noreferrer">Named Entity Recognition</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. The NER feature can identify and categorize entities in unstructured text. For example: people, places, organizations, and quantities. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/language-support?tabs=ga-api" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">language</th><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">en</td><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td align="left">en</td><td align="left">I had a wonderful trip to Seattle last week.</td><td align="left">[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetection">LanguageDetection<a href="#languagedetection" class="hash-link" aria-label="Direct link to LanguageDetection" title="Direct link to LanguageDetection">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/overview" target="_blank" rel="noopener noreferrer">Language detection</a> can detect the language a document is written in. It returns a language code for a wide range of languages, variants, dialects, and some regional/cultural languages. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a document written in English.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;这是一份用中文写的文件&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language_detection </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LanguageDetection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> language_detection</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language.documents.detectedLanguage&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">detectedLanguage</th></tr></thead><tbody><tr><td align="left">This is a document written in English.</td><td align="left">{&quot;name&quot;: &quot;English&quot;, &quot;iso6391Name&quot;: &quot;en&quot;, &quot;confidenceScore&quot;: 0.99}</td></tr><tr><td align="left">这是一份用中文写的文件</td><td align="left">{&quot;name&quot;: &quot;Chinese_Simplified&quot;, &quot;iso6391Name&quot;: &quot;zh_chs&quot;, &quot;confidenceScore&quot;: 1}</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entitylinking">EntityLinking<a href="#entitylinking" class="hash-link" aria-label="Direct link to EntityLinking" title="Direct link to EntityLinking">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/overview" target="_blank" rel="noopener noreferrer">Entity linking</a> identifies and disambiguates the identity of entities found in text. For example, in the sentence &quot;We went to Seattle last week.&quot;, the word &quot;Seattle&quot; would be identified, with a link to more information on Wikipedia. <a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/language-support" target="_blank" rel="noopener noreferrer">English and Spanish are supported</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;We went to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_linking </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityLinking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_linking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[{&quot;bingId&quot;: &quot;a093e9b9-90f5-a3d5-c4b8-5855e1b01f85&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Microsoft&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.48, &quot;length&quot;: 9, &quot;offset&quot;: 0, &quot;text&quot;: &quot;Microsoft&quot;}]<!-- -->, &quot;name&quot;: &quot;Microsoft&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Microsoft%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Microsoft&quot;}</a>, {&quot;bingId&quot;: &quot;0d47c987-0042-5576-15e8-97af601614fa&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Bill Gates&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.52, &quot;length&quot;: 10, &quot;offset&quot;: 25, &quot;text&quot;: &quot;Bill Gates&quot;}]<!-- -->, &quot;name&quot;: &quot;Bill Gates&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Bill_Gates%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Bill_Gates&quot;}</a>, {&quot;bingId&quot;: &quot;df2c4376-9923-6a54-893f-2ee5a5badbc7&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Paul Allen&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.54, &quot;length&quot;: 10, &quot;offset&quot;: 40, &quot;text&quot;: &quot;Paul Allen&quot;}]<!-- -->, &quot;name&quot;: &quot;Paul Allen&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Paul_Allen%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Paul_Allen&quot;}</a>, {&quot;bingId&quot;: &quot;52535f87-235e-b513-54fe-c03e4233ac6e&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;April 4&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.38, &quot;length&quot;: 7, &quot;offset&quot;: 54, &quot;text&quot;: &quot;April 4&quot;}]<!-- -->, &quot;name&quot;: &quot;April 4&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/April_4%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/April_4&quot;}]</a></td></tr><tr><td align="left">We went to Seattle last week.</td><td align="left">[{&quot;bingId&quot;: &quot;5fbba6b8-85e1-4d41-9444-d9055436e473&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Seattle&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.17, &quot;length&quot;: 7, &quot;offset&quot;: 11, &quot;text&quot;: &quot;Seattle&quot;}]<!-- -->, &quot;name&quot;: &quot;Seattle&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Seattle%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Seattle&quot;}]</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextraction">KeyPhraseExtraction<a href="#keyphraseextraction" class="hash-link" aria-label="Direct link to KeyPhraseExtraction" title="Direct link to KeyPhraseExtraction">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/overview" target="_blank" rel="noopener noreferrer">Key phrase extraction</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. Use key phrase extraction to quickly identify the main concepts in text. For example, in the text &quot;The food was delicious and the staff were wonderful.&quot;, key phrase extraction will return the main topics: &quot;food&quot; and &quot;wonderful staff&quot;. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/key-phrase-extraction/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key_phrase_extraction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;KeyPhraseExtraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> key_phrase_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction.documents.keyPhrases&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">keyPhrases</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[&quot;Bill Gates&quot;, &quot;Paul Allen&quot;, &quot;Microsoft&quot;, &quot;April&quot;]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[&quot;modern medical office&quot;, &quot;Dr. Smith&quot;, &quot;great staff&quot;]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="piientityrecognition">PiiEntityRecognition<a href="#piientityrecognition" class="hash-link" aria-label="Direct link to PiiEntityRecognition" title="Direct link to PiiEntityRecognition">​</a></h2><p>The PII detection feature can identify, categorize, and redact sensitive information in unstructured text. For example: phone numbers, email addresses, and forms of identification. The method for utilizing PII in conversations is different than other use cases, and articles for this use have been separated. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/language-support?tabs=documents" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Call our office at 312-555-1234, or send an email to support@contoso.com&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii_entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PiiEntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> pii_entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Call our office at 312-555-1234, or send an email to <a href="mailto:support@contoso.com" target="_blank" rel="noopener noreferrer">support@contoso.com</a></td><td align="left">[{&quot;category&quot;: &quot;PhoneNumber&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 12, &quot;offset&quot;: 19, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;312-555-1234&quot;}, {&quot;category&quot;: &quot;Email&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 19, &quot;offset&quot;: 53, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;support@contoso.com&quot;}]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.93, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sentimentanalysis">SentimentAnalysis<a href="#sentimentanalysis" class="hash-link" aria-label="Direct link to SentimentAnalysis" title="Direct link to SentimentAnalysis">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview" target="_blank" rel="noopener noreferrer">Sentiment analysis</a> and opinion mining are features offered by the Language service, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. These features help you find out what people think of your brand or topic by mining text for clues about positive or negative sentiment, and can associate them with specific aspects of the text. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/sentiment-opinion-mining/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The food and service were unacceptable. The concierge was nice, however.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;It taste great.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment_analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> sentiment_analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">sentiment</th></tr></thead><tbody><tr><td align="left">The food and service were unacceptable. The concierge was nice, however.</td><td align="left">mixed</td></tr><tr><td align="left">It tastes great.</td><td align="left">positive</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="analyze-text-with-textanalyze">Analyze Text with TextAnalyze<a href="#analyze-text-with-textanalyze" class="hash-link" aria-label="Direct link to Analyze Text with TextAnalyze" title="Direct link to Analyze Text with TextAnalyze">​</a></h2><p>Text Analyze is Deprecated, please use AnalyzeText instead</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#named-entity-recognition" class="table-of-contents__link toc-highlight">Named Entity Recognition</a></li><li><a href="#languagedetection" class="table-of-contents__link toc-highlight">LanguageDetection</a></li><li><a href="#entitylinking" class="table-of-contents__link toc-highlight">EntityLinking</a></li><li><a href="#keyphraseextraction" class="table-of-contents__link toc-highlight">KeyPhraseExtraction</a></li><li><a href="#piientityrecognition" class="table-of-contents__link toc-highlight">PiiEntityRecognition</a></li><li><a href="#sentimentanalysis" class="table-of-contents__link toc-highlight">SentimentAnalysis</a></li><li><a href="#analyze-text-with-textanalyze" class="table-of-contents__link toc-highlight">Analyze Text with TextAnalyze</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index 5d4873f4a8..418da781e9 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your Azure AI service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html
index 30b187e1f7..cfdf5f7e5a 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index 5522477f8f..6d47629f3e 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index be8f670ce4..866c269bd2 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index 080d900192..7bac9564bf 100644
--- a/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Azure AI Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure AI Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://learn.microsoft.com/azure/ai-services/multi-service-resource" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector key here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-ai-services-on-spark">Run anomaly detection using AI services on Spark<a href="#run-anomaly-detection-using-ai-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using AI services on Spark" title="Direct link to Run anomaly detection using AI services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitivity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;secondly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unitSymbol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dateTime&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createOrReplaceTempView</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;df_anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">limit 200&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedUpperValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;UpperMargin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Expected Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;royalblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedLowerValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Lower Margin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ro&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM Anomalies with Confidence Intervals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure AI services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-ai-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using AI services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 3eef6f690c..20a12822ac 100644
--- a/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/index.html b/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/index.html
index 8b91f1d5f0..9bb304f87e 100644
--- a/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 4273c0f5d8..226a4c7c42 100644
--- a/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 06c507e7ed..91bcc60a20 100644
--- a/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><img loading="lazy" src="https://camo.githubusercontent.com/4ac8c931fd4600d2b466975c87fb03b439ebc7f6debd58409aea0db10457436d/68747470733a2f2f7777772e6d6963726f736f66742e636f6d2f656e2d75732f72657365617263682f75706c6f6164732f70726f642f323032302f30352f4174747269627574696f6e2e706e67" alt="image-alt-text" class="img_ev3q"></p><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index 4a06cff8ba..2f8574b40d 100644
--- a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index 15e8a2cbaf..575e4ba4d1 100644
--- a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 2c4b6719a6..6605c5a6ce 100644
--- a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 9a636e6a3a..77153a4a5d 100644
--- a/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 2be2b013e2..32a7585a83 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/index.html
index 63e9e5c6e0..2d473182b3 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:1.0.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/index.html
index e78f31b694..d05aa36f7a 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index e098e2e4e3..17a7265600 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index a617cd2dc3..f4adb2afd4 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index 8104648061..fbe72bc9e2 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm onnxmltools</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.7</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index 728c5cd3eb..159834b59f 100644
--- a/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 41bcd3393b..e38ff12696 100644
--- a/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index ab30048f09..99a5310991 100644
--- a/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/LightGBM/Overview/index.html b/docs/1.0.2/Explore Algorithms/LightGBM/Overview/index.html
index 1e79bd8050..7552c5e966 100644
--- a/docs/1.0.2/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/1.0.2/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index a96e8d7d25..d94ce4bd9e 100644
--- a/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/1.0.2/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/index.html b/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/index.html
index b6d42750f3..30a3d6a0bd 100644
--- a/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook into <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a>, <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">Synapse Workspace</a> or if using Databricks into the <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">Databricks Workspace</a>.</li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported.</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><span class="token plain"> pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six unstructured</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.10</span><span class="token number" style="color:rgb(174, 129, 255)">.24</span><span class="token plain"> pytesseract numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><p>Note that you also need to set up your Bing search to gain access to your <a href="https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource" target="_blank" rel="noopener noreferrer">Bing Search subscription key</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index 857791ee22..9bae8dd3f4 100644
--- a/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index dd1bfeefe6..7c977e4d2f 100644
--- a/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in AI Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/OpenAI/index.html b/docs/1.0.2/Explore Algorithms/OpenAI/index.html
index 7a39c4d6c0..21ee4d5005 100644
--- a/docs/1.0.2/Explore Algorithms/OpenAI/index.html	
+++ b/docs/1.0.2/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/index.html
index f8cff56952..05475237f4 100644
--- a/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index 264d5f2aa0..ab281c65c2 100644
--- a/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index 7bcb3490ae..d63ce50bde 100644
--- a/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:1.0.2
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 6b8cd312e7..eaa462c422 100644
--- a/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 061848631e..9b24c669c4 100644
--- a/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index ca5af1ad43..83cca0e3c4 100644
--- a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index 23dd8da219..6342bc14ae 100644
--- a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 62888c6d06..fcb2e3d6ce 100644
--- a/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index f8c349f75a..f30561f941 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index bf6882bee9..dbf31f5069 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/index.html
index ae79817eff..8ce86650a8 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index 802f277652..87418e7200 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index a14b4e7df8..3d98fd2101 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 0815489e64..ee110a0f51 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index e71ea6374f..b781a1c356 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index 6c06ab0bc8..5c3f91a510 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 3039dd70a6..2748d987d6 100644
--- a/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.functions import vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vector_to_array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vector_to_array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index 0783f6e308..0e59f5edcb 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index 8d86a3f8b4..945a65efb4 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index a4fe307703..d6325f9d30 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index 3a4e3447bd..b564ab726b 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index ad53c6a336..782043aa67 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 171c464794..1dfdb66d3f 100644
--- a/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Get Started/Create a Spark Cluster/index.html b/docs/1.0.2/Get Started/Create a Spark Cluster/index.html
index 3de9d47933..95507e689b 100644
--- a/docs/1.0.2/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/1.0.2/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Get Started/Install SynapseML/index.html b/docs/1.0.2/Get Started/Install SynapseML/index.html
index 81bf31f5aa..589ceb47c2 100644
--- a/docs/1.0.2/Get Started/Install SynapseML/index.html	
+++ b/docs/1.0.2/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/1.0.2/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Get Started/Quickstart - Your First Models/index.html b/docs/1.0.2/Get Started/Quickstart - Your First Models/index.html
index 82c00de057..758c80bae1 100644
--- a/docs/1.0.2/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/1.0.2/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-azure-ai-services-to-transform-data-in-one-step">Use Azure AI Services to transform data in one step<a href="#use-azure-ai-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Azure AI Services to transform data in one step" title="Direct link to Use Azure AI Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Azure AI Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-azure-ai-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Azure AI Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Get Started/Set up Cognitive Services/index.html b/docs/1.0.2/Get Started/Set up Cognitive Services/index.html
index c425df117e..0e222514e8 100644
--- a/docs/1.0.2/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/1.0.2/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Azure AI Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/en-us/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Azure AI Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create Azure AI service (Decision, Language, Speech, Vision) resource</a>. You can follow the steps at <a href="https://learn.microsoft.com/en-us/azure/ai-services/multi-service-resource?tabs=windows&amp;pivots=azportal#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">Create a multi-service resource for Azure AI services</a>. </li><li>Get your Azure AI Services resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret">Manage secrets with Azure Key Vault and access the secrets with find_secret<a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="hash-link" aria-label="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret" title="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret">​</a></h2><p>After you create an Azure AI resource, you will obtain a resource key. You can use this resource key directly in our notebooks as a string, but we recommend to manage secrets with Azure Key Vault. Azure Key Vault is a cloud-based service that allows you to store and manage cryptographic keys, certificates, and secrets used by cloud applications and services.</p><p>You can skip the following content if you want to use a secret string instead of Azure Key Vault. This is not recommended for production workloads.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-key-vault">Create Azure Key Vault<a href="#create-azure-key-vault" class="hash-link" aria-label="Direct link to Create Azure Key Vault" title="Direct link to Create Azure Key Vault">​</a></h3><p>Refer to <a href="https://learn.microsoft.com/azure/key-vault/general/quick-create-portal" target="_blank" rel="noopener noreferrer">this article</a> to create a key vault using Azure Portal</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-secret-to-azure-key-vault">Save secret to Azure Key Vault<a href="#save-secret-to-azure-key-vault" class="hash-link" aria-label="Direct link to Save secret to Azure Key Vault" title="Direct link to Save secret to Azure Key Vault">​</a></h3><ul><li>Go to Access control (IAM) and assign <strong>Key Vault Administrator Role</strong> to yourself.</li><li>Go to Secrets and choose <strong>+ Generate/Import</strong>, create a key with the secret value obtained from Azure AI service.</li><li>Choose <strong>Create</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-microsoft-fabric--power-bi">Use find_secret on Microsoft Fabric / Power BI<a href="#use-find_secret-on-microsoft-fabric--power-bi" class="hash-link" aria-label="Direct link to Use find_secret on Microsoft Fabric / Power BI" title="Direct link to Use find_secret on Microsoft Fabric / Power BI">​</a></h3><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Granted the Fabric or Power BI Service Account Get permissions in the Azure Key Vault.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-azure-databricks">Use find_secret on Azure Databricks<a href="#use-find_secret-on-azure-databricks" class="hash-link" aria-label="Direct link to Use find_secret on Azure Databricks" title="Direct link to Use find_secret on Azure Databricks">​</a></h3><ul><li>In the Azure Portal, find and select the Azure Key Vault Instance. Select the <strong>Access configuration</strong> tab under Settings. Set Permission model to Vault access policy.</li><li>On Databricks, go to <code>https://&lt;databricks-instance&gt;#secrets/createScope</code>. This URL is case sensitive.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_create_secret_scope.png" width="600" class="img_ev3q"><ul><li>Enter the name of the secret scope and choose desired Manage Principal.</li><li>Go to your Azure Key Vault -&gt; Properties and find the <strong>DNS Name</strong> (Vault URI on Azure Key Vault) and <strong>Resource ID</strong>. Enter the DNS Name and Resource ID on Databricks createScope page.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_find_resource_id.png" width="600" class="img_ev3q"><ul><li>Click the <strong>Create</strong> Button.</li></ul><p>Refer to <a href="https://learn.microsoft.com/azure/databricks/security/secrets/secret-scopes#--create-an-azure-key-vault-backed-secret-scope" target="_blank" rel="noopener noreferrer">this article</a> for more details.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-synapse">Use find_secret on Synapse<a href="#use-find_secret-on-synapse" class="hash-link" aria-label="Direct link to Use find_secret on Synapse" title="Direct link to Use find_secret on Synapse">​</a></h3><ul><li>Open the Synapse Studio and go to the Manage tab.</li><li>Under External connections, select Linked services.</li><li>To add a linked service, select New.</li><li>Select the Azure Key Vault tile from the list and select Continue.</li><li>Enter a linked service name and choose the key vault you want to connect to.</li><li>Select Create</li><li>Click Publish</li></ul><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Choose <strong>Key Vault Administrator</strong> in Role blade, select Next.</li><li>In Members blade, choose Assign access to <strong>Managed identity</strong>. Select members, choose the subscription your Synapse Workspace in. For Managed identity, select Synapse workspace, choose your workspace.</li><li>Select <strong>Review + assign</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="quick-test">Quick Test<a href="#quick-test" class="hash-link" aria-label="Direct link to Quick Test" title="Direct link to Quick Test">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own key vault name and api name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own AI service location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your result should looks like</p><table><thead><tr><th>language</th><th>text</th><th>entities</th></tr></thead><tbody><tr><td>en</td><td>Dr. Smith has a very modern medical office, and she has great staff.</td><td>[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td>en</td><td>I had a wonderful trip to Seattle last week.</td><td>[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li><li><a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="table-of-contents__link toc-highlight">Manage secrets with Azure Key Vault and access the secrets with find_secret</a><ul><li><a href="#create-azure-key-vault" class="table-of-contents__link toc-highlight">Create Azure Key Vault</a></li><li><a href="#save-secret-to-azure-key-vault" class="table-of-contents__link toc-highlight">Save secret to Azure Key Vault</a></li><li><a href="#use-find_secret-on-microsoft-fabric--power-bi" class="table-of-contents__link toc-highlight">Use find_secret on Microsoft Fabric / Power BI</a></li><li><a href="#use-find_secret-on-azure-databricks" class="table-of-contents__link toc-highlight">Use find_secret on Azure Databricks</a></li><li><a href="#use-find_secret-on-synapse" class="table-of-contents__link toc-highlight">Use find_secret on Synapse</a></li><li><a href="#quick-test" class="table-of-contents__link toc-highlight">Quick Test</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Overview/index.html b/docs/1.0.2/Overview/index.html
index 75683a57d2..35c5465da5 100644
--- a/docs/1.0.2/Overview/index.html
+++ b/docs/1.0.2/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/estimators/estimators_causal/index.html b/docs/1.0.2/Quick Examples/estimators/estimators_causal/index.html
index 70ed7dbf54..e1fdd74059 100644
--- a/docs/1.0.2/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/1.0.2/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/index.html b/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/index.html
index e53309e13b..9c518b8c84 100644
--- a/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/estimators/estimators_core/index.html b/docs/1.0.2/Quick Examples/estimators/estimators_core/index.html
index f6fecd495e..60bf1e4c81 100644
--- a/docs/1.0.2/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/1.0.2/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/index.html
index 97a14d9c0b..80ca2977c1 100644
--- a/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/estimators/estimators_vw/index.html b/docs/1.0.2/Quick Examples/estimators/estimators_vw/index.html
index 4513bf2009..0693e9b80f 100644
--- a/docs/1.0.2/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/1.0.2/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/index.html b/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/index.html
index ea00187c02..1ede2b1971 100644
--- a/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1, p._2, 1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .++(baseSeq.map(p =&gt; (p._1, p._2, 2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;timestamp&quot;, &quot;value&quot;, &quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i * imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/transformers/transformers_core/index.html b/docs/1.0.2/Quick Examples/transformers/transformers_core/index.html
index 388560b082..9706315327 100644
--- a/docs/1.0.2/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/1.0.2/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/index.html
index 99445264ca..a1bd25d464 100644
--- a/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/transformers/transformers_opencv/index.html b/docs/1.0.2/Quick Examples/transformers/transformers_opencv/index.html
index 9c5d253a65..21275314ba 100644
--- a/docs/1.0.2/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/1.0.2/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Quick Examples/transformers/transformers_vw/index.html b/docs/1.0.2/Quick Examples/transformers/transformers_vw/index.html
index 4ea20fb329..8f8b8a92ec 100644
--- a/docs/1.0.2/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/1.0.2/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/Contributor Guide/index.html b/docs/1.0.2/Reference/Contributor Guide/index.html
index 99d2b95482..f81dc33d65 100644
--- a/docs/1.0.2/Reference/Contributor Guide/index.html	
+++ b/docs/1.0.2/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/Developer Setup/index.html b/docs/1.0.2/Reference/Developer Setup/index.html
index 342e70e751..7fe8891693 100644
--- a/docs/1.0.2/Reference/Developer Setup/index.html	
+++ b/docs/1.0.2/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/Docker Setup/index.html b/docs/1.0.2/Reference/Docker Setup/index.html
index 5d45a03097..dd551e9d61 100644
--- a/docs/1.0.2/Reference/Docker Setup/index.html	
+++ b/docs/1.0.2/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/Dotnet Setup/index.html b/docs/1.0.2/Reference/Dotnet Setup/index.html
index e0dce5485a..8dd805f061 100644
--- a/docs/1.0.2/Reference/Dotnet Setup/index.html	
+++ b/docs/1.0.2/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The Azure AI services on spark aint bad|   en-US| null|[{positive, null, {0.99, 0.01, 0.00}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.2/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/index.html
index fafdef6fd0..7458627f0c 100644
--- a/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/1.0.2/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Reference/R Setup/index.html b/docs/1.0.2/Reference/R Setup/index.html
index 354e26c54c..37f1d21a2f 100644
--- a/docs/1.0.2/Reference/R Setup/index.html	
+++ b/docs/1.0.2/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Use with MLFlow/Autologging/index.html b/docs/1.0.2/Use with MLFlow/Autologging/index.html
index 1245f027c7..c1b5952ed7 100644
--- a/docs/1.0.2/Use with MLFlow/Autologging/index.html	
+++ b/docs/1.0.2/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Use with MLFlow/Install/index.html b/docs/1.0.2/Use with MLFlow/Install/index.html
index 4fb9298ec8..bfaf396e44 100644
--- a/docs/1.0.2/Use with MLFlow/Install/index.html	
+++ b/docs/1.0.2/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.2/Use with MLFlow/Overview/index.html b/docs/1.0.2/Use with MLFlow/Overview/index.html
index 9343754cb4..dc43efdef3 100644
--- a/docs/1.0.2/Use with MLFlow/Overview/index.html	
+++ b/docs/1.0.2/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.2/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.2/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.2</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.2</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Deploy Models/Overview/index.html b/docs/1.0.3/Deploy Models/Overview/index.html
index e606ecefa7..04cf5823dd 100644
--- a/docs/1.0.3/Deploy Models/Overview/index.html	
+++ b/docs/1.0.3/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/index.html
index 18a7bda90c..33bd1c0766 100644
--- a/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 0280b96bbb..c8cb08a35a 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>AI Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the AI service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.language import AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The AI services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKind(&quot;SentimentAnalysis&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.documents.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key-2&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/index.html
index 96871442e8..df9aef69ba 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/overview-cognitive-services#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index 20726637e4..f0c42bd6b7 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: AI Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure AI services on Apache Spark to detect anomalies in multivariate time-series data. Multivariate anomaly detection takes correlations and dependencies between the different variables into account when discovering anomalies. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI Services, and we then use to the model to detect anomalies in a  dataset of synthetic measurements from three IoT sensors. </p><p>To learn more about the Azure AI Anomaly Detector, refer to <a href="https://docs.microsoft.com/azure/ai-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><p>For Spark3.3 and below versions, the output of select statements will be in the format of <code>List&lt;Rows&gt;</code>, so to format the data into dictionary and generate the values when interpretation is empty, please use the below parse method:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def parse(x):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if len(x) &gt; 0:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return dict([item[:2] for item in x])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return {&quot;sensor_1&quot;: 0, &quot;sensor_2&quot;: 0, &quot;sensor_3&quot;: 0}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Staring with Spark3.4, the output of the select statement is already formatted as a <code>numpy.ndarry&lt;dictionary&gt;</code> and no need to format the data again, so please use below parse method to generate the values when interpretation is empty:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Overview/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Overview/index.html
index 95fce545f4..87bf882d6a 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure AI Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><p><a href="https://azure.microsoft.com/services/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI services</a> is a suite of APIs, SDKs, and services that developers can use to add intelligent features to their applications. AI services empower developers even when they don&#x27;t have direct AI or data science skills or knowledge. Azure AI services help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure AI services can be categorized into five main pillars: Vision, Speech, Language, Web search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-language" target="_blank" rel="noopener noreferrer"><strong>AI Language</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/services-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.bing.html#module-synapse.ml.services.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/services-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Azure AI services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-your-system">Prepare your system<a href="#prepare-your-system" class="hash-link" aria-label="Direct link to Prepare your system" title="Direct link to Prepare your system">​</a></h2><p>To begin, import required libraries and initialize your Spark session.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Import Azure AI services libraries and replace the keys in the following code snippet with your Azure AI services key.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general AI services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-sentiment-analysis-on-text">Perform sentiment analysis on text<a href="#perform-sentiment-analysis-on-text" class="hash-link" aria-label="Direct link to Perform sentiment analysis on text" title="Direct link to Perform sentiment analysis on text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-language/" target="_blank" rel="noopener noreferrer">AI Language</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The AI services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-text-analytics-for-health-data">Perform text analytics for health data<a href="#perform-text-analytics-for-health-data" class="hash-link" aria-label="Direct link to Perform text analytics for health data" title="Direct link to Perform text analytics for health data">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/ai-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured text such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><p>The following code sample analyzes and transforms text from doctors notes into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translate-text-into-a-different-language">Translate text into a different language<a href="#translate-text-into-a-different-language" class="hash-link" aria-label="Direct link to Translate text into a different language" title="Direct link to Translate text into a different language">​</a></h2><p><a href="https://azure.microsoft.com/services/ai-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure AI services family of AI APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used to translate text without hosting your own algorithm.</p><p>The following code sample does a simple text translation by providing the sentences you want to translate and target languages you want to translate them to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="extract-information-from-a-document-into-structured-data">Extract information from a document into structured data<a href="#extract-information-from-a-document-into-structured-data" class="hash-link" aria-label="Direct link to Extract information from a document into structured data" title="Direct link to Extract information from a document into structured data">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. With Form Recognizer, you can identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more.</p><p>The following code sample analyzes a business card image and extracts its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-vision/" target="_blank" rel="noopener noreferrer">Azure AI Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions.</p><p>The following code sample analyzes images and labels them with tags. Tags are one-word descriptions of things in the image, such as recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://www.microsoft.com/bing/apis/bing-image-search-api" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query.</p><p>The following code sample uses a text query that looks for images with quotes. The output of the code is a list of image URLs that contain photos related to the query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-speech-to-text">Transform speech to text<a href="#transform-speech-to-text" class="hash-link" aria-label="Direct link to Transform speech to text" title="Direct link to Transform speech to text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-speech/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. The following code sample transcribes one audio file to text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/text-to-speech/" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows you to build applications that speak naturally. You can choose from more than 270 neural voices across 119 languages.</p><p>The following code sample transforms text into an audio file that contains the content of the text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="detect-anomalies-in-time-series-data">Detect anomalies in time series data<a href="#detect-anomalies-in-time-series-data" class="hash-link" aria-label="Direct link to Detect anomalies in time series data" title="Direct link to Detect anomalies in time series data">​</a></h2><p>If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one. You may want to skip this part.</p><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. The following code sample uses the Anomaly Detector service to find anomalies in a time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-information-from-arbitrary-web-apis">Get information from arbitrary web APIs<a href="#get-information-from-arbitrary-web-apis" class="hash-link" aria-label="Direct link to Get information from arbitrary web APIs" title="Direct link to Get information from arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#prepare-your-system" class="table-of-contents__link toc-highlight">Prepare your system</a></li><li><a href="#perform-sentiment-analysis-on-text" class="table-of-contents__link toc-highlight">Perform sentiment analysis on text</a></li><li><a href="#perform-text-analytics-for-health-data" class="table-of-contents__link toc-highlight">Perform text analytics for health data</a></li><li><a href="#translate-text-into-a-different-language" class="table-of-contents__link toc-highlight">Translate text into a different language</a></li><li><a href="#extract-information-from-a-document-into-structured-data" class="table-of-contents__link toc-highlight">Extract information from a document into structured data</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#transform-speech-to-text" class="table-of-contents__link toc-highlight">Transform speech to text</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#detect-anomalies-in-time-series-data" class="table-of-contents__link toc-highlight">Detect anomalies in time series data</a></li><li><a href="#get-information-from-arbitrary-web-apis" class="table-of-contents__link toc-highlight">Get information from arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index 1aad7d93d8..0bc127693d 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Azure AI Services</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index 6c2c04d8ad..2abcc72b7f 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Analyze Text with SynapseML and Azure AI Language</h1><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/overview" target="_blank" rel="noopener noreferrer">Azure AI Language</a> is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.
 You can use SynapseML with Azure AI Language for <strong>named entity recognition</strong>, <strong>language detection</strong>, <strong>entity linking</strong>, <strong>key phrase extraction</strong>, <strong>Pii entity recognition</strong> and <strong>sentiment analysis</strong>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="named-entity-recognition">Named Entity Recognition<a href="#named-entity-recognition" class="hash-link" aria-label="Direct link to Named Entity Recognition" title="Direct link to Named Entity Recognition">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/overview" target="_blank" rel="noopener noreferrer">Named Entity Recognition</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. The NER feature can identify and categorize entities in unstructured text. For example: people, places, organizations, and quantities. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/language-support?tabs=ga-api" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">language</th><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">en</td><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td align="left">en</td><td align="left">I had a wonderful trip to Seattle last week.</td><td align="left">[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetection">LanguageDetection<a href="#languagedetection" class="hash-link" aria-label="Direct link to LanguageDetection" title="Direct link to LanguageDetection">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/overview" target="_blank" rel="noopener noreferrer">Language detection</a> can detect the language a document is written in. It returns a language code for a wide range of languages, variants, dialects, and some regional/cultural languages. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a document written in English.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;这是一份用中文写的文件&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language_detection </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LanguageDetection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> language_detection</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language.documents.detectedLanguage&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">detectedLanguage</th></tr></thead><tbody><tr><td align="left">This is a document written in English.</td><td align="left">{&quot;name&quot;: &quot;English&quot;, &quot;iso6391Name&quot;: &quot;en&quot;, &quot;confidenceScore&quot;: 0.99}</td></tr><tr><td align="left">这是一份用中文写的文件</td><td align="left">{&quot;name&quot;: &quot;Chinese_Simplified&quot;, &quot;iso6391Name&quot;: &quot;zh_chs&quot;, &quot;confidenceScore&quot;: 1}</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entitylinking">EntityLinking<a href="#entitylinking" class="hash-link" aria-label="Direct link to EntityLinking" title="Direct link to EntityLinking">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/overview" target="_blank" rel="noopener noreferrer">Entity linking</a> identifies and disambiguates the identity of entities found in text. For example, in the sentence &quot;We went to Seattle last week.&quot;, the word &quot;Seattle&quot; would be identified, with a link to more information on Wikipedia. <a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/language-support" target="_blank" rel="noopener noreferrer">English and Spanish are supported</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;We went to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_linking </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityLinking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_linking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[{&quot;bingId&quot;: &quot;a093e9b9-90f5-a3d5-c4b8-5855e1b01f85&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Microsoft&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.48, &quot;length&quot;: 9, &quot;offset&quot;: 0, &quot;text&quot;: &quot;Microsoft&quot;}]<!-- -->, &quot;name&quot;: &quot;Microsoft&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Microsoft%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Microsoft&quot;}</a>, {&quot;bingId&quot;: &quot;0d47c987-0042-5576-15e8-97af601614fa&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Bill Gates&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.52, &quot;length&quot;: 10, &quot;offset&quot;: 25, &quot;text&quot;: &quot;Bill Gates&quot;}]<!-- -->, &quot;name&quot;: &quot;Bill Gates&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Bill_Gates%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Bill_Gates&quot;}</a>, {&quot;bingId&quot;: &quot;df2c4376-9923-6a54-893f-2ee5a5badbc7&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Paul Allen&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.54, &quot;length&quot;: 10, &quot;offset&quot;: 40, &quot;text&quot;: &quot;Paul Allen&quot;}]<!-- -->, &quot;name&quot;: &quot;Paul Allen&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Paul_Allen%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Paul_Allen&quot;}</a>, {&quot;bingId&quot;: &quot;52535f87-235e-b513-54fe-c03e4233ac6e&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;April 4&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.38, &quot;length&quot;: 7, &quot;offset&quot;: 54, &quot;text&quot;: &quot;April 4&quot;}]<!-- -->, &quot;name&quot;: &quot;April 4&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/April_4%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/April_4&quot;}]</a></td></tr><tr><td align="left">We went to Seattle last week.</td><td align="left">[{&quot;bingId&quot;: &quot;5fbba6b8-85e1-4d41-9444-d9055436e473&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Seattle&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.17, &quot;length&quot;: 7, &quot;offset&quot;: 11, &quot;text&quot;: &quot;Seattle&quot;}]<!-- -->, &quot;name&quot;: &quot;Seattle&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Seattle%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Seattle&quot;}]</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextraction">KeyPhraseExtraction<a href="#keyphraseextraction" class="hash-link" aria-label="Direct link to KeyPhraseExtraction" title="Direct link to KeyPhraseExtraction">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/overview" target="_blank" rel="noopener noreferrer">Key phrase extraction</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. Use key phrase extraction to quickly identify the main concepts in text. For example, in the text &quot;The food was delicious and the staff were wonderful.&quot;, key phrase extraction will return the main topics: &quot;food&quot; and &quot;wonderful staff&quot;. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/key-phrase-extraction/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key_phrase_extraction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;KeyPhraseExtraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> key_phrase_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction.documents.keyPhrases&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">keyPhrases</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[&quot;Bill Gates&quot;, &quot;Paul Allen&quot;, &quot;Microsoft&quot;, &quot;April&quot;]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[&quot;modern medical office&quot;, &quot;Dr. Smith&quot;, &quot;great staff&quot;]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="piientityrecognition">PiiEntityRecognition<a href="#piientityrecognition" class="hash-link" aria-label="Direct link to PiiEntityRecognition" title="Direct link to PiiEntityRecognition">​</a></h2><p>The PII detection feature can identify, categorize, and redact sensitive information in unstructured text. For example: phone numbers, email addresses, and forms of identification. The method for utilizing PII in conversations is different than other use cases, and articles for this use have been separated. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/language-support?tabs=documents" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Call our office at 312-555-1234, or send an email to support@contoso.com&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii_entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PiiEntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> pii_entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Call our office at 312-555-1234, or send an email to <a href="mailto:support@contoso.com" target="_blank" rel="noopener noreferrer">support@contoso.com</a></td><td align="left">[{&quot;category&quot;: &quot;PhoneNumber&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 12, &quot;offset&quot;: 19, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;312-555-1234&quot;}, {&quot;category&quot;: &quot;Email&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 19, &quot;offset&quot;: 53, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;support@contoso.com&quot;}]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.93, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sentimentanalysis">SentimentAnalysis<a href="#sentimentanalysis" class="hash-link" aria-label="Direct link to SentimentAnalysis" title="Direct link to SentimentAnalysis">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview" target="_blank" rel="noopener noreferrer">Sentiment analysis</a> and opinion mining are features offered by the Language service, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. These features help you find out what people think of your brand or topic by mining text for clues about positive or negative sentiment, and can associate them with specific aspects of the text. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/sentiment-opinion-mining/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The food and service were unacceptable. The concierge was nice, however.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;It taste great.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment_analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> sentiment_analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">sentiment</th></tr></thead><tbody><tr><td align="left">The food and service were unacceptable. The concierge was nice, however.</td><td align="left">mixed</td></tr><tr><td align="left">It tastes great.</td><td align="left">positive</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="analyze-text-with-textanalyze">Analyze Text with TextAnalyze<a href="#analyze-text-with-textanalyze" class="hash-link" aria-label="Direct link to Analyze Text with TextAnalyze" title="Direct link to Analyze Text with TextAnalyze">​</a></h2><p>Text Analyze is Deprecated, please use AnalyzeText instead</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#named-entity-recognition" class="table-of-contents__link toc-highlight">Named Entity Recognition</a></li><li><a href="#languagedetection" class="table-of-contents__link toc-highlight">LanguageDetection</a></li><li><a href="#entitylinking" class="table-of-contents__link toc-highlight">EntityLinking</a></li><li><a href="#keyphraseextraction" class="table-of-contents__link toc-highlight">KeyPhraseExtraction</a></li><li><a href="#piientityrecognition" class="table-of-contents__link toc-highlight">PiiEntityRecognition</a></li><li><a href="#sentimentanalysis" class="table-of-contents__link toc-highlight">SentimentAnalysis</a></li><li><a href="#analyze-text-with-textanalyze" class="table-of-contents__link toc-highlight">Analyze Text with TextAnalyze</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index bde0546a52..f93599fe31 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your Azure AI service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html
index 9e8560784b..33f9d69238 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index 9bbd11df52..63c05d1e19 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index e3becd349a..9fc4cb817f 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index 5c765f17d4..07ac954c64 100644
--- a/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Azure AI Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure AI Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://learn.microsoft.com/azure/ai-services/multi-service-resource" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector key here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-ai-services-on-spark">Run anomaly detection using AI services on Spark<a href="#run-anomaly-detection-using-ai-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using AI services on Spark" title="Direct link to Run anomaly detection using AI services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitivity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;secondly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unitSymbol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dateTime&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createOrReplaceTempView</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;df_anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">limit 200&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedUpperValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;UpperMargin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Expected Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;royalblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedLowerValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Lower Margin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ro&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM Anomalies with Confidence Intervals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure AI services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-ai-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using AI services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 81c8bf3190..ba40b9f3e2 100644
--- a/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/index.html b/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/index.html
index ee7cba0f2d..e78f6a7533 100644
--- a/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 2fa0f79f00..dcc74099c7 100644
--- a/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 888df8d54e..f034f1eae3 100644
--- a/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index 7246a7bd5d..819a1f3b54 100644
--- a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index 333fb3e502..01b04f1611 100644
--- a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index e77fba5e1b..6b4f7e5c03 100644
--- a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index e37d466370..6fd929cde5 100644
--- a/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 353f5466ce..648ee9745f 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/index.html
index 4b1e8ff2e3..db5710ea3b 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:1.0.3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/index.html
index bb056028c1..11a30f7a39 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index 23244a4bea..bacb83e3e4 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index 1afcfd4866..0613493fb0 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index f1010df016..16536ade52 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm onnxmltools</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.7</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index d120dbdf85..c11c745439 100644
--- a/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 42a1937205..17ff9ea1f1 100644
--- a/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index 530d484e34..f525279491 100644
--- a/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/LightGBM/Overview/index.html b/docs/1.0.3/Explore Algorithms/LightGBM/Overview/index.html
index 4b816c8c3a..ea6ae64b35 100644
--- a/docs/1.0.3/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/1.0.3/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index 76e832ecda..66c7d66e70 100644
--- a/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/1.0.3/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/index.html b/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/index.html
index da108f66df..283161c79a 100644
--- a/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook into <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a>, <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">Synapse Workspace</a> or if using Databricks into the <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">Databricks Workspace</a>.</li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported.</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><span class="token plain"> pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six unstructured</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.10</span><span class="token number" style="color:rgb(174, 129, 255)">.24</span><span class="token plain"> pytesseract numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><p>Note that you also need to set up your Bing search to gain access to your <a href="https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource" target="_blank" rel="noopener noreferrer">Bing Search subscription key</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index f0baa55130..5b9edbcc23 100644
--- a/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index ff3ac0e5b4..fe6e1ae2ee 100644
--- a/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in AI Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/OpenAI/index.html b/docs/1.0.3/Explore Algorithms/OpenAI/index.html
index cbf6cb196d..b45be3ca54 100644
--- a/docs/1.0.3/Explore Algorithms/OpenAI/index.html	
+++ b/docs/1.0.3/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/index.html
index 669a7941a5..4bedd729fd 100644
--- a/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index 474de05069..6b0426bd02 100644
--- a/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index 43d2bef03a..50d7088ae8 100644
--- a/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:1.0.3
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 57e05c8156..42aeb54d1a 100644
--- a/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 5be9eb75c9..e66a2763d5 100644
--- a/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index 6b1d5bea8b..43fa2443e5 100644
--- a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index 40cb17445c..86b723522c 100644
--- a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 937392d7c9..c7edb8edc1 100644
--- a/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index d2a3663779..0ff3988018 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index 3a982b7164..c4a3fb6605 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/index.html
index 24e3a0742c..6f4ad58ede 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index b5a782d503..4135ff1195 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index a87a4ec157..d59cbe3914 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 96e8b14655..4dbb13ef3a 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index 9ba8480e6b..d4e8be33f3 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index c1e20ffabe..a2f567ea10 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 04d2ace0bb..8979c86c8d 100644
--- a/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.functions import vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vector_to_array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vector_to_array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index c268b6259a..1c9197591e 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index 9228cd6609..2edbdab00b 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index 1c6b1dab16..ee72e47b53 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index 0de5ecb0f4..6f3dd68c75 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index bfa4605aa2..c9bf73d161 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 9d0292073f..511784450c 100644
--- a/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Get Started/Create a Spark Cluster/index.html b/docs/1.0.3/Get Started/Create a Spark Cluster/index.html
index 8238954fb4..5886a31c10 100644
--- a/docs/1.0.3/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/1.0.3/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Get Started/Install SynapseML/index.html b/docs/1.0.3/Get Started/Install SynapseML/index.html
index 3e8df40755..0ba0f5f29b 100644
--- a/docs/1.0.3/Get Started/Install SynapseML/index.html	
+++ b/docs/1.0.3/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/1.0.3/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Get Started/Quickstart - Your First Models/index.html b/docs/1.0.3/Get Started/Quickstart - Your First Models/index.html
index fec333b7a1..01640c4207 100644
--- a/docs/1.0.3/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/1.0.3/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-azure-ai-services-to-transform-data-in-one-step">Use Azure AI Services to transform data in one step<a href="#use-azure-ai-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Azure AI Services to transform data in one step" title="Direct link to Use Azure AI Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Azure AI Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-azure-ai-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Azure AI Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Get Started/Set up Cognitive Services/index.html b/docs/1.0.3/Get Started/Set up Cognitive Services/index.html
index 751365224f..0e18a4be19 100644
--- a/docs/1.0.3/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/1.0.3/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Azure AI Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/en-us/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Azure AI Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create Azure AI service (Decision, Language, Speech, Vision) resource</a>. You can follow the steps at <a href="https://learn.microsoft.com/en-us/azure/ai-services/multi-service-resource?tabs=windows&amp;pivots=azportal#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">Create a multi-service resource for Azure AI services</a>. </li><li>Get your Azure AI Services resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret">Manage secrets with Azure Key Vault and access the secrets with find_secret<a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="hash-link" aria-label="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret" title="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret">​</a></h2><p>After you create an Azure AI resource, you will obtain a resource key. You can use this resource key directly in our notebooks as a string, but we recommend to manage secrets with Azure Key Vault. Azure Key Vault is a cloud-based service that allows you to store and manage cryptographic keys, certificates, and secrets used by cloud applications and services.</p><p>You can skip the following content if you want to use a secret string instead of Azure Key Vault. This is not recommended for production workloads.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-key-vault">Create Azure Key Vault<a href="#create-azure-key-vault" class="hash-link" aria-label="Direct link to Create Azure Key Vault" title="Direct link to Create Azure Key Vault">​</a></h3><p>Refer to <a href="https://learn.microsoft.com/azure/key-vault/general/quick-create-portal" target="_blank" rel="noopener noreferrer">this article</a> to create a key vault using Azure Portal</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-secret-to-azure-key-vault">Save secret to Azure Key Vault<a href="#save-secret-to-azure-key-vault" class="hash-link" aria-label="Direct link to Save secret to Azure Key Vault" title="Direct link to Save secret to Azure Key Vault">​</a></h3><ul><li>Go to Access control (IAM) and assign <strong>Key Vault Administrator Role</strong> to yourself.</li><li>Go to Secrets and choose <strong>+ Generate/Import</strong>, create a key with the secret value obtained from Azure AI service.</li><li>Choose <strong>Create</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-microsoft-fabric--power-bi">Use find_secret on Microsoft Fabric / Power BI<a href="#use-find_secret-on-microsoft-fabric--power-bi" class="hash-link" aria-label="Direct link to Use find_secret on Microsoft Fabric / Power BI" title="Direct link to Use find_secret on Microsoft Fabric / Power BI">​</a></h3><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Granted the Fabric or Power BI Service Account Get permissions in the Azure Key Vault.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-azure-databricks">Use find_secret on Azure Databricks<a href="#use-find_secret-on-azure-databricks" class="hash-link" aria-label="Direct link to Use find_secret on Azure Databricks" title="Direct link to Use find_secret on Azure Databricks">​</a></h3><ul><li>In the Azure Portal, find and select the Azure Key Vault Instance. Select the <strong>Access configuration</strong> tab under Settings. Set Permission model to Vault access policy.</li><li>On Databricks, go to <code>https://&lt;databricks-instance&gt;#secrets/createScope</code>. This URL is case sensitive.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_create_secret_scope.png" width="600" class="img_ev3q"><ul><li>Enter the name of the secret scope and choose desired Manage Principal.</li><li>Go to your Azure Key Vault -&gt; Properties and find the <strong>DNS Name</strong> (Vault URI on Azure Key Vault) and <strong>Resource ID</strong>. Enter the DNS Name and Resource ID on Databricks createScope page.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_find_resource_id.png" width="600" class="img_ev3q"><ul><li>Click the <strong>Create</strong> Button.</li></ul><p>Refer to <a href="https://learn.microsoft.com/azure/databricks/security/secrets/secret-scopes#--create-an-azure-key-vault-backed-secret-scope" target="_blank" rel="noopener noreferrer">this article</a> for more details.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-synapse">Use find_secret on Synapse<a href="#use-find_secret-on-synapse" class="hash-link" aria-label="Direct link to Use find_secret on Synapse" title="Direct link to Use find_secret on Synapse">​</a></h3><ul><li>Open the Synapse Studio and go to the Manage tab.</li><li>Under External connections, select Linked services.</li><li>To add a linked service, select New.</li><li>Select the Azure Key Vault tile from the list and select Continue.</li><li>Enter a linked service name and choose the key vault you want to connect to.</li><li>Select Create</li><li>Click Publish</li></ul><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Choose <strong>Key Vault Administrator</strong> in Role blade, select Next.</li><li>In Members blade, choose Assign access to <strong>Managed identity</strong>. Select members, choose the subscription your Synapse Workspace in. For Managed identity, select Synapse workspace, choose your workspace.</li><li>Select <strong>Review + assign</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="quick-test">Quick Test<a href="#quick-test" class="hash-link" aria-label="Direct link to Quick Test" title="Direct link to Quick Test">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own key vault name and api name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own AI service location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your result should looks like</p><table><thead><tr><th>language</th><th>text</th><th>entities</th></tr></thead><tbody><tr><td>en</td><td>Dr. Smith has a very modern medical office, and she has great staff.</td><td>[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td>en</td><td>I had a wonderful trip to Seattle last week.</td><td>[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li><li><a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="table-of-contents__link toc-highlight">Manage secrets with Azure Key Vault and access the secrets with find_secret</a><ul><li><a href="#create-azure-key-vault" class="table-of-contents__link toc-highlight">Create Azure Key Vault</a></li><li><a href="#save-secret-to-azure-key-vault" class="table-of-contents__link toc-highlight">Save secret to Azure Key Vault</a></li><li><a href="#use-find_secret-on-microsoft-fabric--power-bi" class="table-of-contents__link toc-highlight">Use find_secret on Microsoft Fabric / Power BI</a></li><li><a href="#use-find_secret-on-azure-databricks" class="table-of-contents__link toc-highlight">Use find_secret on Azure Databricks</a></li><li><a href="#use-find_secret-on-synapse" class="table-of-contents__link toc-highlight">Use find_secret on Synapse</a></li><li><a href="#quick-test" class="table-of-contents__link toc-highlight">Quick Test</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Overview/index.html b/docs/1.0.3/Overview/index.html
index e7c86b8676..3a02334f6a 100644
--- a/docs/1.0.3/Overview/index.html
+++ b/docs/1.0.3/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/estimators/estimators_causal/index.html b/docs/1.0.3/Quick Examples/estimators/estimators_causal/index.html
index 959239cc82..c684b32966 100644
--- a/docs/1.0.3/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/1.0.3/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/index.html b/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/index.html
index 37d80cb2da..3753273fe7 100644
--- a/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/estimators/estimators_core/index.html b/docs/1.0.3/Quick Examples/estimators/estimators_core/index.html
index c6b71d4789..05a4a71a67 100644
--- a/docs/1.0.3/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/1.0.3/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/index.html
index 8e78d48476..4493075317 100644
--- a/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/estimators/estimators_vw/index.html b/docs/1.0.3/Quick Examples/estimators/estimators_vw/index.html
index 3bbaeb7637..cb6c5b1cbd 100644
--- a/docs/1.0.3/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/1.0.3/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/index.html b/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/index.html
index 807eb002e7..4bfd61cce5 100644
--- a/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1, p._2, 1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .++(baseSeq.map(p =&gt; (p._1, p._2, 2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;timestamp&quot;, &quot;value&quot;, &quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i * imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/transformers/transformers_core/index.html b/docs/1.0.3/Quick Examples/transformers/transformers_core/index.html
index 815ba45366..fb84c48250 100644
--- a/docs/1.0.3/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/1.0.3/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/index.html
index d42b117f4b..7907a32574 100644
--- a/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/transformers/transformers_opencv/index.html b/docs/1.0.3/Quick Examples/transformers/transformers_opencv/index.html
index 6be53ba7bf..ee1f68294a 100644
--- a/docs/1.0.3/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/1.0.3/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Quick Examples/transformers/transformers_vw/index.html b/docs/1.0.3/Quick Examples/transformers/transformers_vw/index.html
index 70449ca3e1..b2baedcca7 100644
--- a/docs/1.0.3/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/1.0.3/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/Contributor Guide/index.html b/docs/1.0.3/Reference/Contributor Guide/index.html
index 0ea01c173d..ea357728a7 100644
--- a/docs/1.0.3/Reference/Contributor Guide/index.html	
+++ b/docs/1.0.3/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/Developer Setup/index.html b/docs/1.0.3/Reference/Developer Setup/index.html
index 87817d7d8b..f90905999f 100644
--- a/docs/1.0.3/Reference/Developer Setup/index.html	
+++ b/docs/1.0.3/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/Docker Setup/index.html b/docs/1.0.3/Reference/Docker Setup/index.html
index 7adca94db3..1885620fd9 100644
--- a/docs/1.0.3/Reference/Docker Setup/index.html	
+++ b/docs/1.0.3/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/Dotnet Setup/index.html b/docs/1.0.3/Reference/Dotnet Setup/index.html
index 58ce4bcc22..a240e19b48 100644
--- a/docs/1.0.3/Reference/Dotnet Setup/index.html	
+++ b/docs/1.0.3/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The Azure AI services on spark aint bad|   en-US| null|[{positive, null, {0.99, 0.01, 0.00}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.3/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/index.html
index 740f6e73a2..be71e237c2 100644
--- a/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/1.0.3/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Reference/R Setup/index.html b/docs/1.0.3/Reference/R Setup/index.html
index 44fc849ed0..f0ee9c90f4 100644
--- a/docs/1.0.3/Reference/R Setup/index.html	
+++ b/docs/1.0.3/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Use with MLFlow/Autologging/index.html b/docs/1.0.3/Use with MLFlow/Autologging/index.html
index 7c4919947f..85a51c011d 100644
--- a/docs/1.0.3/Use with MLFlow/Autologging/index.html	
+++ b/docs/1.0.3/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Use with MLFlow/Install/index.html b/docs/1.0.3/Use with MLFlow/Install/index.html
index 36c5226255..ab81ee3f2d 100644
--- a/docs/1.0.3/Use with MLFlow/Install/index.html	
+++ b/docs/1.0.3/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/1.0.3/Use with MLFlow/Overview/index.html b/docs/1.0.3/Use with MLFlow/Overview/index.html
index c4ff1d6af8..5843bb6c79 100644
--- a/docs/1.0.3/Use with MLFlow/Overview/index.html	
+++ b/docs/1.0.3/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/1.0.3/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/1.0.3/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is documentation for <!-- -->SynapseML<!-- --> <b>1.0.3</b>, which is no longer actively maintained.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.3</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Deploy Models/Overview/index.html b/docs/Deploy Models/Overview/index.html
index 5b4dfef189..4e3eda5112 100644
--- a/docs/Deploy Models/Overview/index.html	
+++ b/docs/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/Deploy Models/Quickstart - Deploying a Classifier/index.html
index c4c8e6160c..3bdd474c57 100644
--- a/docs/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 95984af6f1..6907556f0e 100644
--- a/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>AI Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the AI service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.language import AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The AI services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKind(&quot;SentimentAnalysis&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.documents.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key-2&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/Explore Algorithms/AI Services/Geospatial Services/index.html
index ae4ec69b35..ac17bcad56 100644
--- a/docs/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/overview-cognitive-services#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index bd20419111..4d9ca305e8 100644
--- a/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: AI Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure AI services on Apache Spark to detect anomalies in multivariate time-series data. Multivariate anomaly detection takes correlations and dependencies between the different variables into account when discovering anomalies. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI Services, and we then use to the model to detect anomalies in a  dataset of synthetic measurements from three IoT sensors. </p><p>To learn more about the Azure AI Anomaly Detector, refer to <a href="https://docs.microsoft.com/azure/ai-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, click <code>Create</code> in your resource group, and then type <code>Anomaly Detector</code>. Click on the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then click <code>Review + Create</code> and then <code>Create</code>.</li><li>Once the Anomaly Detector resource is created, open it and click on the <code>Keys and Endpoints</code> panel on the left. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s now format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><p>For Spark3.3 and below versions, the output of select statements will be in the format of <code>List&lt;Rows&gt;</code>, so to format the data into dictionary and generate the values when interpretation is empty, please use the below parse method:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def parse(x):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if len(x) &gt; 0:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return dict([item[:2] for item in x])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return {&quot;sensor_1&quot;: 0, &quot;sensor_2&quot;: 0, &quot;sensor_3&quot;: 0}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Staring with Spark3.4, the output of the select statement is already formatted as a <code>numpy.ndarry&lt;dictionary&gt;</code> and no need to format the data again, so please use below parse method to generate the values when interpretation is empty:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Let&#x27;s run the next cell to plot the results. The <code>minSeverity</code> parameter in the first line specifies the minimum severity of the anomalies to be plotted. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Overview/index.html b/docs/Explore Algorithms/AI Services/Overview/index.html
index 924b253078..921c9c3914 100644
--- a/docs/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure AI Services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><p><a href="https://azure.microsoft.com/services/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI services</a> is a suite of APIs, SDKs, and services that developers can use to add intelligent features to their applications. AI services empower developers even when they don&#x27;t have direct AI or data science skills or knowledge. Azure AI services help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure AI services can be categorized into five main pillars: Vision, Speech, Language, Web search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/speech-services/" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-language" target="_blank" rel="noopener noreferrer"><strong>AI Language</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/translator/" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h3><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer"><strong>Form Recognizer</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/services-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer">Bing Image search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.bing.html#module-synapse.ml.services.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive search</a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/services-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Azure AI services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-your-system">Prepare your system<a href="#prepare-your-system" class="hash-link" aria-label="Direct link to Prepare your system" title="Direct link to Prepare your system">​</a></h2><p>To begin, import required libraries and initialize your Spark session.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Import Azure AI services libraries and replace the keys in the following code snippet with your Azure AI services key.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general AI services key for Text Analytics, Computer Vision and Form Recognizer (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-sentiment-analysis-on-text">Perform sentiment analysis on text<a href="#perform-sentiment-analysis-on-text" class="hash-link" aria-label="Direct link to Perform sentiment analysis on text" title="Direct link to Perform sentiment analysis on text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-language/" target="_blank" rel="noopener noreferrer">AI Language</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The AI services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-text-analytics-for-health-data">Perform text analytics for health data<a href="#perform-text-analytics-for-health-data" class="hash-link" aria-label="Direct link to Perform text analytics for health data" title="Direct link to Perform text analytics for health data">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/ai-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured text such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><p>The following code sample analyzes and transforms text from doctors notes into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translate-text-into-a-different-language">Translate text into a different language<a href="#translate-text-into-a-different-language" class="hash-link" aria-label="Direct link to Translate text into a different language" title="Direct link to Translate text into a different language">​</a></h2><p><a href="https://azure.microsoft.com/services/ai-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure AI services family of AI APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used to translate text without hosting your own algorithm.</p><p>The following code sample does a simple text translation by providing the sentences you want to translate and target languages you want to translate them to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="extract-information-from-a-document-into-structured-data">Extract information from a document into structured data<a href="#extract-information-from-a-document-into-structured-data" class="hash-link" aria-label="Direct link to Extract information from a document into structured data" title="Direct link to Extract information from a document into structured data">​</a></h2><p><a href="https://azure.microsoft.com/services/form-recognizer/" target="_blank" rel="noopener noreferrer">Form Recognizer</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. With Form Recognizer, you can identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more.</p><p>The following code sample analyzes a business card image and extracts its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-vision/" target="_blank" rel="noopener noreferrer">Azure AI Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions.</p><p>The following code sample analyzes images and labels them with tags. Tags are one-word descriptions of things in the image, such as recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search-sample">Bing Image Search sample<a href="#bing-image-search-sample" class="hash-link" aria-label="Direct link to Bing Image Search sample" title="Direct link to Bing Image Search sample">​</a></h2><p><a href="https://www.microsoft.com/bing/apis/bing-image-search-api" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query.</p><p>The following code sample uses a text query that looks for images with quotes. The output of the code is a list of image URLs that contain photos related to the query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-speech-to-text">Transform speech to text<a href="#transform-speech-to-text" class="hash-link" aria-label="Direct link to Transform speech to text" title="Direct link to Transform speech to text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-speech/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. The following code sample transcribes one audio file to text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-to-speech-sample">Text-to-Speech sample<a href="#text-to-speech-sample" class="hash-link" aria-label="Direct link to Text-to-Speech sample" title="Direct link to Text-to-Speech sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/text-to-speech/" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows you to build applications that speak naturally. You can choose from more than 270 neural voices across 119 languages.</p><p>The following code sample transforms text into an audio file that contains the content of the text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="detect-anomalies-in-time-series-data">Detect anomalies in time series data<a href="#detect-anomalies-in-time-series-data" class="hash-link" aria-label="Direct link to Detect anomalies in time series data" title="Direct link to Detect anomalies in time series data">​</a></h2><p>If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one. You may want to skip this part.</p><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. The following code sample uses the Anomaly Detector service to find anomalies in a time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-information-from-arbitrary-web-apis">Get information from arbitrary web APIs<a href="#get-information-from-arbitrary-web-apis" class="hash-link" aria-label="Direct link to Get information from arbitrary web APIs" title="Direct link to Get information from arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-cognitive-search-sample">Azure Cognitive search sample<a href="#azure-cognitive-search-sample" class="hash-link" aria-label="Direct link to Azure Cognitive search sample" title="Direct link to Azure Cognitive search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#prepare-your-system" class="table-of-contents__link toc-highlight">Prepare your system</a></li><li><a href="#perform-sentiment-analysis-on-text" class="table-of-contents__link toc-highlight">Perform sentiment analysis on text</a></li><li><a href="#perform-text-analytics-for-health-data" class="table-of-contents__link toc-highlight">Perform text analytics for health data</a></li><li><a href="#translate-text-into-a-different-language" class="table-of-contents__link toc-highlight">Translate text into a different language</a></li><li><a href="#extract-information-from-a-document-into-structured-data" class="table-of-contents__link toc-highlight">Extract information from a document into structured data</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#bing-image-search-sample" class="table-of-contents__link toc-highlight">Bing Image Search sample</a></li><li><a href="#transform-speech-to-text" class="table-of-contents__link toc-highlight">Transform speech to text</a></li><li><a href="#text-to-speech-sample" class="table-of-contents__link toc-highlight">Text-to-Speech sample</a></li><li><a href="#detect-anomalies-in-time-series-data" class="table-of-contents__link toc-highlight">Detect anomalies in time series data</a></li><li><a href="#get-information-from-arbitrary-web-apis" class="table-of-contents__link toc-highlight">Get information from arbitrary web APIs</a></li><li><a href="#azure-cognitive-search-sample" class="table-of-contents__link toc-highlight">Azure Cognitive search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index 4591189d00..4f20f02e65 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Azure AI Services</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index 9d12499cac..dee0ea1691 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Analyze Text with SynapseML and Azure AI Language</h1><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/overview" target="_blank" rel="noopener noreferrer">Azure AI Language</a> is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.
 You can use SynapseML with Azure AI Language for <strong>named entity recognition</strong>, <strong>language detection</strong>, <strong>entity linking</strong>, <strong>key phrase extraction</strong>, <strong>Pii entity recognition</strong> and <strong>sentiment analysis</strong>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="named-entity-recognition">Named Entity Recognition<a href="#named-entity-recognition" class="hash-link" aria-label="Direct link to Named Entity Recognition" title="Direct link to Named Entity Recognition">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/overview" target="_blank" rel="noopener noreferrer">Named Entity Recognition</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. The NER feature can identify and categorize entities in unstructured text. For example: people, places, organizations, and quantities. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/language-support?tabs=ga-api" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">language</th><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">en</td><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td align="left">en</td><td align="left">I had a wonderful trip to Seattle last week.</td><td align="left">[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetection">LanguageDetection<a href="#languagedetection" class="hash-link" aria-label="Direct link to LanguageDetection" title="Direct link to LanguageDetection">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/overview" target="_blank" rel="noopener noreferrer">Language detection</a> can detect the language a document is written in. It returns a language code for a wide range of languages, variants, dialects, and some regional/cultural languages. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a document written in English.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;这是一份用中文写的文件&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language_detection </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LanguageDetection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> language_detection</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language.documents.detectedLanguage&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">detectedLanguage</th></tr></thead><tbody><tr><td align="left">This is a document written in English.</td><td align="left">{&quot;name&quot;: &quot;English&quot;, &quot;iso6391Name&quot;: &quot;en&quot;, &quot;confidenceScore&quot;: 0.99}</td></tr><tr><td align="left">这是一份用中文写的文件</td><td align="left">{&quot;name&quot;: &quot;Chinese_Simplified&quot;, &quot;iso6391Name&quot;: &quot;zh_chs&quot;, &quot;confidenceScore&quot;: 1}</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entitylinking">EntityLinking<a href="#entitylinking" class="hash-link" aria-label="Direct link to EntityLinking" title="Direct link to EntityLinking">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/overview" target="_blank" rel="noopener noreferrer">Entity linking</a> identifies and disambiguates the identity of entities found in text. For example, in the sentence &quot;We went to Seattle last week.&quot;, the word &quot;Seattle&quot; would be identified, with a link to more information on Wikipedia. <a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/language-support" target="_blank" rel="noopener noreferrer">English and Spanish are supported</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;We went to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_linking </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityLinking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_linking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[{&quot;bingId&quot;: &quot;a093e9b9-90f5-a3d5-c4b8-5855e1b01f85&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Microsoft&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.48, &quot;length&quot;: 9, &quot;offset&quot;: 0, &quot;text&quot;: &quot;Microsoft&quot;}]<!-- -->, &quot;name&quot;: &quot;Microsoft&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Microsoft%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Microsoft&quot;}</a>, {&quot;bingId&quot;: &quot;0d47c987-0042-5576-15e8-97af601614fa&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Bill Gates&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.52, &quot;length&quot;: 10, &quot;offset&quot;: 25, &quot;text&quot;: &quot;Bill Gates&quot;}]<!-- -->, &quot;name&quot;: &quot;Bill Gates&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Bill_Gates%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Bill_Gates&quot;}</a>, {&quot;bingId&quot;: &quot;df2c4376-9923-6a54-893f-2ee5a5badbc7&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Paul Allen&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.54, &quot;length&quot;: 10, &quot;offset&quot;: 40, &quot;text&quot;: &quot;Paul Allen&quot;}]<!-- -->, &quot;name&quot;: &quot;Paul Allen&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Paul_Allen%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Paul_Allen&quot;}</a>, {&quot;bingId&quot;: &quot;52535f87-235e-b513-54fe-c03e4233ac6e&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;April 4&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.38, &quot;length&quot;: 7, &quot;offset&quot;: 54, &quot;text&quot;: &quot;April 4&quot;}]<!-- -->, &quot;name&quot;: &quot;April 4&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/April_4%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/April_4&quot;}]</a></td></tr><tr><td align="left">We went to Seattle last week.</td><td align="left">[{&quot;bingId&quot;: &quot;5fbba6b8-85e1-4d41-9444-d9055436e473&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Seattle&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.17, &quot;length&quot;: 7, &quot;offset&quot;: 11, &quot;text&quot;: &quot;Seattle&quot;}]<!-- -->, &quot;name&quot;: &quot;Seattle&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Seattle%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Seattle&quot;}]</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextraction">KeyPhraseExtraction<a href="#keyphraseextraction" class="hash-link" aria-label="Direct link to KeyPhraseExtraction" title="Direct link to KeyPhraseExtraction">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/overview" target="_blank" rel="noopener noreferrer">Key phrase extraction</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. Use key phrase extraction to quickly identify the main concepts in text. For example, in the text &quot;The food was delicious and the staff were wonderful.&quot;, key phrase extraction will return the main topics: &quot;food&quot; and &quot;wonderful staff&quot;. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/key-phrase-extraction/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key_phrase_extraction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;KeyPhraseExtraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> key_phrase_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction.documents.keyPhrases&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">keyPhrases</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[&quot;Bill Gates&quot;, &quot;Paul Allen&quot;, &quot;Microsoft&quot;, &quot;April&quot;]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[&quot;modern medical office&quot;, &quot;Dr. Smith&quot;, &quot;great staff&quot;]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="piientityrecognition">PiiEntityRecognition<a href="#piientityrecognition" class="hash-link" aria-label="Direct link to PiiEntityRecognition" title="Direct link to PiiEntityRecognition">​</a></h2><p>The PII detection feature can identify, categorize, and redact sensitive information in unstructured text. For example: phone numbers, email addresses, and forms of identification. The method for utilizing PII in conversations is different than other use cases, and articles for this use have been separated. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/language-support?tabs=documents" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Call our office at 312-555-1234, or send an email to support@contoso.com&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii_entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PiiEntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> pii_entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Call our office at 312-555-1234, or send an email to <a href="mailto:support@contoso.com" target="_blank" rel="noopener noreferrer">support@contoso.com</a></td><td align="left">[{&quot;category&quot;: &quot;PhoneNumber&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 12, &quot;offset&quot;: 19, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;312-555-1234&quot;}, {&quot;category&quot;: &quot;Email&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 19, &quot;offset&quot;: 53, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;support@contoso.com&quot;}]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.93, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sentimentanalysis">SentimentAnalysis<a href="#sentimentanalysis" class="hash-link" aria-label="Direct link to SentimentAnalysis" title="Direct link to SentimentAnalysis">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview" target="_blank" rel="noopener noreferrer">Sentiment analysis</a> and opinion mining are features offered by the Language service, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. These features help you find out what people think of your brand or topic by mining text for clues about positive or negative sentiment, and can associate them with specific aspects of the text. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/sentiment-opinion-mining/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The food and service were unacceptable. The concierge was nice, however.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;It taste great.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment_analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> sentiment_analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">sentiment</th></tr></thead><tbody><tr><td align="left">The food and service were unacceptable. The concierge was nice, however.</td><td align="left">mixed</td></tr><tr><td align="left">It tastes great.</td><td align="left">positive</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="analyze-text-with-textanalyze">Analyze Text with TextAnalyze<a href="#analyze-text-with-textanalyze" class="hash-link" aria-label="Direct link to Analyze Text with TextAnalyze" title="Direct link to Analyze Text with TextAnalyze">​</a></h2><p>Text Analyze is Deprecated, please use AnalyzeText instead</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#named-entity-recognition" class="table-of-contents__link toc-highlight">Named Entity Recognition</a></li><li><a href="#languagedetection" class="table-of-contents__link toc-highlight">LanguageDetection</a></li><li><a href="#entitylinking" class="table-of-contents__link toc-highlight">EntityLinking</a></li><li><a href="#keyphraseextraction" class="table-of-contents__link toc-highlight">KeyPhraseExtraction</a></li><li><a href="#piientityrecognition" class="table-of-contents__link toc-highlight">PiiEntityRecognition</a></li><li><a href="#sentimentanalysis" class="table-of-contents__link toc-highlight">SentimentAnalysis</a></li><li><a href="#analyze-text-with-textanalyze" class="table-of-contents__link toc-highlight">Analyze Text with TextAnalyze</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index ebfca87c4b..e6d385e011 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your Azure AI service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://marhamilresearch4.blob.core.windows.net/gutenberg-public/Notebook/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html
index ff4654e25d..2513bb0322 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index 03039041b5..7d92f354ea 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://public@synapseaisolutionsa.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index d87e330b4b..4ec64c63cf 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index f5423bfe8f..6e72ab565c 100644
--- a/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Azure AI Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure AI Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://learn.microsoft.com/azure/ai-services/multi-service-resource" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector key here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-ai-services-on-spark">Run anomaly detection using AI services on Spark<a href="#run-anomaly-detection-using-ai-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using AI services on Spark" title="Direct link to Run anomaly detection using AI services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitivity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;secondly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unitSymbol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dateTime&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createOrReplaceTempView</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;df_anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">limit 200&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedUpperValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;UpperMargin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Expected Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;royalblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedLowerValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Lower Margin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ro&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM Anomalies with Confidence Intervals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure AI services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-ai-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using AI services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 094d1f8dc1..7769ce0fc1 100644
--- a/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Multivariate Anomaly Detection with Isolation Forest</h1><p>This recipe shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If you are running it on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a>.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolation-forest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># model_version = 1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global_feature_importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Causal Inference/Overview/index.html b/docs/Explore Algorithms/Causal Inference/Overview/index.html
index c2dd5853a7..54bcea9d6a 100644
--- a/docs/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 031a0cfef5..d676217bf9 100644
--- a/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Causal Effects with SynapseML DoubleMLEstimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index a55e25e398..70e9762ffe 100644
--- a/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Synthetic difference in differences</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html b/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html
index 053754080d..f93ff9279d 100644
--- a/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html	
+++ b/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Synthetic difference in differences</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="theme-doc-markdown markdown"><h1>Scalable Synthetic Difference in Differences</h1><p>This sample notebook aims to show readers how to use SynapseML&#x27;s <code>DiffInDiffEstimator</code>, <code>SyntheticControlEstimator</code> and <code>SyntheticDiffInDiffEstimator</code> to estimate the causal effect of a treatment on a particular outcome.</p><p>In this sample notebook, we will use the California smoking cessation program example to demonstrate usage of the SyntheticDiffInDiff Estimator. The goal of the analysis is to estimate the effect of increased cigarette taxes on smoking in California.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SyntheticControlEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SyntheticDiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib </span><span class="token keyword">import</span><span class="token plain"> pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib </span><span class="token keyword">import</span><span class="token plain"> style</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLogLevel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;INFO&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">style</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">use</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ggplot&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We will select 5 columns from the dataset: state, year, cigsale, california, after_treatment.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/smoking.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we use the <code>DiffInDiffEstimator</code> to estimate the causal effect with regular difference in differences method. We set the treatment indicator column to &quot;california&quot;, set post-treatment indicator column to &quot;after_treatment&quot;, and set the outcome column to &quot;cigsale&quot;.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator1 </span><span class="token operator">=</span><span class="token plain"> DiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model1 </span><span class="token operator">=</span><span class="token plain"> estimator1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Diff in Diff] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Diff in Diff] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by difference in differences should be -27.349.</p><p>Next, we use <code>SyntheticControlEstimator</code> to synthesize a control unit and use the synthetic control to estimate the causal effect. To create the synthetic control unit, we need to set the column which indicates the time when each outcome is measured, and the column which indicates the unit for which the outcome is measured.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator2 </span><span class="token operator">=</span><span class="token plain"> SyntheticControlEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    timeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    unitCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterNoChange</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tol</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1e-4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stepSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model2 </span><span class="token operator">=</span><span class="token plain"> estimator2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Control] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Control] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by synthetic control should be about -19.354.</p><p>Internally, a constrained least square regression is used to solve the unit weights for the synthetic control, and we can plot the loss history.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryUnitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - unit weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also visualize the synthetic control and compare it with the treated unit.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc_weights </span><span class="token operator">=</span><span class="token plain"> model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdf </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;~california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;California&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control Estimation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cigarette Sales&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1988</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ymin</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ymax</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">140</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Proposition 99&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Lastly, we use <code>SyntheticDiffInDiffEstimator</code> to estimate the causal effect.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator3 </span><span class="token operator">=</span><span class="token plain"> SyntheticDiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    timeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    unitCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterNoChange</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tol</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1e-4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stepSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model3 </span><span class="token operator">=</span><span class="token plain"> estimator3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Diff in Diff] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Diff in Diff] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by synthetic control should be about -15.554.</p><p>Again, we can plot the loss history from the optimizer used to solve the unit weights and the time weights.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryUnitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - unit weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryTimeWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - time weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Here we plot the synthetic diff in diff estimate together with the time weights.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">unit_weights </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unit_intercept </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitIntercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time_weights </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">timeWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time_intercept </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">timeIntercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdf </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pivot_df_control </span><span class="token operator">=</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;~california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pivot_df_treat </span><span class="token operator">=</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc_did </span><span class="token operator">=</span><span class="token plain"> pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values @ unit_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treated_mean </span><span class="token operator">=</span><span class="token plain"> pivot_df_treat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gridspec_kw</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;height_ratios&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suptitle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Diff in Diff Estimation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dashed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Control Avg.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;California&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sc_did</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control (SDID)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cigarette Sales&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1989</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dotted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Prop. 99&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;skyblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Time Weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Time&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1989</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dotted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index 7082364b4b..97f46be2dd 100644
--- a/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index 646a18512d..d46be6c94a 100644
--- a/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;abfss://synapse@mmlsparkeuap.dfs.core.windows.net/models/AdultCensus.mml&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Synthetic difference in differences</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 7fe2e43f35..767f50fa20 100644
--- a/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 769c4b10c3..b81d6bcd91 100644
--- a/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 847e1902a8..acfc68d89d 100644
--- a/docs/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/Explore Algorithms/Deep Learning/Getting Started/index.html
index 55d581755b..ff80ab0dbc 100644
--- a/docs/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:1.0.4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/Explore Algorithms/Deep Learning/ONNX/index.html
index 2a54b91839..954c6d6056 100644
--- a/docs/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index 6ed737e591..6a7f8d2498 100644
--- a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index 89fd4213f4..018d3ac84d 100644
--- a/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/17flowers/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index ac2fda88b3..db6bd53a7c 100644
--- a/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm onnxmltools</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.7</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates testing data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index 2593f4ff78..0a1beadfb7 100644
--- a/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index 630aea3ff3..dd348cb3d7 100644
--- a/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index 9c7a4cecdb..250b545c94 100644
--- a/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/LightGBM/Overview/index.html b/docs/Explore Algorithms/LightGBM/Overview/index.html
index 782a5f5e9e..b3ef6133ad 100644
--- a/docs/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index 48b14df219..c40b713bf6 100644
--- a/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong> LightGBM on Spark is available on Spark, PySpark, and SparklyR</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage:<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage:" title="Direct link to LightGBM Usage:">​</a></h3><ul><li>LightGBMClassifier: used for building classification models. For example, to predict whether a company enters bankruptcy or not, we could build a binary classification model with LightGBMClassifier.</li><li>LightGBMRegressor: used for building regression models. For example, to predict the house price, we could build a regression model with LightGBMRegressor.</li><li>LightGBMRanker: used for building ranking models. For example, to predict website searching result relevance, we could build a ranking model with LightGBMRanker.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bankruptcy-prediction-with-lightgbm-classifier">Bankruptcy Prediction with LightGBM Classifier<a href="#bankruptcy-prediction-with-lightgbm-classifier" class="hash-link" aria-label="Direct link to Bankruptcy Prediction with LightGBM Classifier" title="Direct link to Bankruptcy Prediction with LightGBM Classifier">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="add-featurizer-to-convert-features-to-vector">Add featurizer to convert features to vector<a href="#add-featurizer-to-convert-features-to-vector" class="hash-link" aria-label="Direct link to Add featurizer to convert features to vector" title="Direct link to Add featurizer to convert features to vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="feature-importances-visualization">Feature Importances Visualization<a href="#feature-importances-visualization" class="hash-link" aria-label="Direct link to Feature Importances Visualization" title="Direct link to Feature Importances Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-lightgbmregressor">Quantile Regression for Drug Discovery with LightGBMRegressor<a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor" title="Direct link to Quantile Regression for Drug Discovery with LightGBMRegressor">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-ranker">LightGBM Ranker<a href="#lightgbm-ranker" class="hash-link" aria-label="Direct link to LightGBM Ranker" title="Direct link to LightGBM Ranker">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage:</a></li><li><a href="#bankruptcy-prediction-with-lightgbm-classifier" class="table-of-contents__link toc-highlight">Bankruptcy Prediction with LightGBM Classifier</a></li><li><a href="#quantile-regression-for-drug-discovery-with-lightgbmregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with LightGBMRegressor</a></li><li><a href="#lightgbm-ranker" class="table-of-contents__link toc-highlight">LightGBM Ranker</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenAI/Langchain/index.html b/docs/Explore Algorithms/OpenAI/Langchain/index.html
index 5d6a1f7c0b..34f0a52735 100644
--- a/docs/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook into <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a>, <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">Synapse Workspace</a> or if using Databricks into the <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">Databricks Workspace</a>.</li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported.</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><span class="token plain"> pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six unstructured</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.10</span><span class="token number" style="color:rgb(174, 129, 255)">.24</span><span class="token plain"> pytesseract numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><p>Note that you also need to set up your Bing search to gain access to your <a href="https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource" target="_blank" rel="noopener noreferrer">Bing Search subscription key</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-davinci-003&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html b/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html
index 54e58444db..d50cb5bf78 100644
--- a/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html	
+++ b/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding and GPU based KNN</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI and GPU based KNN</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://github.com/NVIDIA/spark-rapids-ml/" target="_blank" rel="noopener noreferrer">Spark Rapids ML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on &quot;Quickstart - OpenAI Embedding&quot; notebook with NVIDIA GPU accelerated KNN.</p><p><strong>Note</strong>: Running the notebook with the demo dataset (Step 4) will generate the same results as CPU based “Quickstart - OpenAI Embedding” notebook. To see  GPU acceleration you need to run query against bigger embeddings.
 For example, running 100K rows dataset will give 6x acceleration and consume less than 10x memory on 2 nodes NVIDIA T4 cluster compare to AMD Epic (Rome) 2 nodes CPU cluster.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>To run the notebook on Databricks add related init script (/tools/init_scripts/init-rapidsml-cuda-11.8.sh)</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-query-against-embeddings">Step 8: Build the query against embeddings<a href="#step-8-build-the-query-against-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build the query against embeddings" title="Direct link to Step 8: Build the query against embeddings">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-fit-knn-model">Step 9: Fit KNN model<a href="#step-9-fit-knn-model" class="hash-link" aria-label="Direct link to Step 9: Fit KNN model" title="Direct link to Step 9: Fit KNN model">​</a></h2><p>Build KNN model using fit method</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> spark_rapids_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">knn </span><span class="token keyword">import</span><span class="token plain"> NearestNeighbors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn </span><span class="token operator">=</span><span class="token plain"> NearestNeighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn_model </span><span class="token operator">=</span><span class="token plain"> rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-10-retrieve-query-results">Step 10: Retrieve query results<a href="#step-10-retrieve-query-results" class="hash-link" aria-label="Direct link to Step 10: Retrieve query results" title="Direct link to Step 10: Retrieve query results">​</a></h2><p>Find k nearest neighbors</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rapids_knn_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">kneighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding_query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-the-query-against-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build the query against embeddings</a></li><li><a href="#step-9-fit-knn-model" class="table-of-contents__link toc-highlight">Step 9: Fit KNN model</a></li><li><a href="#step-10-retrieve-query-results" class="table-of-contents__link toc-highlight">Step 10: Retrieve query results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index 2b0e12e0a8..e2859a73ad 100644
--- a/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index 7ffd28b390..035e9a10d9 100644
--- a/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You will set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://ignite2021@mmlsparkdemo.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlsparkdemo.blob.core.windows.net/ignite2021/form_svgs/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the AnalyzeInvoices transformer and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/0.10.0/pyspark/synapse.ml.cognitive.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads Translate, a transformer that calls the Azure Translator service in AI Services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenAI/index.html b/docs/Explore Algorithms/OpenAI/index.html
index d3915457ea..1fa33fa3f9 100644
--- a/docs/Explore Algorithms/OpenAI/index.html	
+++ b/docs/Explore Algorithms/OpenAI/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure Open AI and Azure Synapse Analytics. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. If using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. If using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li><li>Connect your notebook to a cluster and follow along, editing and running the cells.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings_3 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-3-small&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-create-a-dataset-of-prompts">Step 4: Create a dataset of prompts<a href="#step-4-create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Step 4: Create a dataset of prompts" title="Direct link to Step 4: Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-create-the-openaicompletion-apache-spark-client">Step 5: Create the OpenAICompletion Apache Spark Client<a href="#step-5-create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client" title="Direct link to Step 5: Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-transform-the-dataframe-with-the-openaicompletion-client">Step 6: Transform the dataframe with the OpenAICompletion Client<a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client" title="Direct link to Step 6: Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>Now that you have the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. We&#x27;ll select out just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. Please note completion text will be different</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="additional-usage-examples">Additional Usage Examples<a href="#additional-usage-examples" class="hash-link" aria-label="Direct link to Additional Usage Examples" title="Direct link to Additional Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task. For more information on using <code>OpenAIEmbedding</code>, see our <a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings-with-reduced-dimensions">Generating Text Embeddings with Reduced Dimensions<a href="#generating-text-embeddings-with-reduced-dimensions" class="hash-link" aria-label="Direct link to Generating Text Embeddings with Reduced Dimensions" title="Direct link to Generating Text Embeddings with Reduced Dimensions">​</a></h3><p>Text-Embedding-3 models developed by OpenAI are trained using a Matryoshka Representation Learning technique
 which supports reducing the dimension of the embedding by trading-off some performance.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings_3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setApiVersion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2024-03-01-preview&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDimensions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there is currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform a request will then be made per row. Since there are multiple prompts in a single row, each is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Step 4: Create a dataset of prompts</a></li><li><a href="#step-5-create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Step 5: Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#step-6-transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Step 6: Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#additional-usage-examples" class="table-of-contents__link toc-highlight">Additional Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#generating-text-embeddings-with-reduced-dimensions" class="table-of-contents__link toc-highlight">Generating Text Embeddings with Reduced Dimensions</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/Explore Algorithms/OpenCV/Image Transformations/index.html
index fd4be21fdc..a4c2f63c7e 100644
--- a/docs/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index d749d15330..82fb24f07e 100644
--- a/docs/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index a3e1c2b749..9f42e10b25 100644
--- a/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:1.0.4
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 0ac423764c..dfd4909286 100644
--- a/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 3e1c36c2a1..ce1dccbda0 100644
--- a/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index 4a93392d6a..a4b081a54e 100644
--- a/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index b248f83056..11f51c11ba 100644
--- a/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 35964551d0..1af0dbd8b0 100644
--- a/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index b0ab4e188f..10337c6cb0 100644
--- a/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index 8f7d1f7789..0eac0be4f6 100644
--- a/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/Explore Algorithms/Responsible AI/Image Explainers/index.html
index 3281fc7b01..265efb5972 100644
--- a/docs/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index 821d50dac8..bc5c0fde0c 100644
--- a/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index ef5ab3ec47..077bec633a 100644
--- a/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 28553c82f8..68a6c706ee 100644
--- a/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index 854b16892b..356457085b 100644
--- a/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index 1f3a062a5b..e47d3c0834 100644
--- a/docs/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 25cc08987a..69137aa1e3 100644
--- a/docs/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.functions import vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vector_to_array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vector_to_array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index 45512a120b..c2222c0fb6 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index ffe2d0700d..d32b8b0b70 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index 8722cde1c9..8cdeda08ec 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index e82557ca00..332110d4b6 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index 66b3eaf771..8ba12883a5 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 56aef522b8..6275a9de4d 100644
--- a/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Get Started/Create a Spark Cluster/index.html b/docs/Get Started/Create a Spark Cluster/index.html
index d1424cd00f..319b0d5cb8 100644
--- a/docs/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Get Started/Install SynapseML/index.html b/docs/Get Started/Install SynapseML/index.html
index a062802ddb..7e59bdb8f1 100644
--- a/docs/Get Started/Install SynapseML/index.html	
+++ b/docs/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -44,7 +44,7 @@
 and some necessary custom wrappers may be missing.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="c-net">C# (.NET)<a href="#c-net" class="hash-link" aria-label="Direct link to C# (.NET)" title="Direct link to C# (.NET)">​</a></h2><p>To try out SynapseML with .NET, follow the <a href="/SynapseML/docs/Reference/Dotnet Setup/">.NET Installation Guide</a>.
 Note: Some stages including AzureSearchWriter, DiagnosticInfo, UDPyF Param, ParamSpaceParam, BallTreeParam,
 ConditionalBallTreeParam, LightGBMBooster Param are still under development and not exposed in .NET.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li><li><a href="#c-net" class="table-of-contents__link toc-highlight">C# (.NET)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Get Started/Quickstart - Your First Models/index.html b/docs/Get Started/Quickstart - Your First Models/index.html
index d5d13de546..9ec73d0959 100644
--- a/docs/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-azure-ai-services-to-transform-data-in-one-step">Use Azure AI Services to transform data in one step<a href="#use-azure-ai-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Azure AI Services to transform data in one step" title="Direct link to Use Azure AI Services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Azure AI Services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-azure-ai-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Azure AI Services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Get Started/Set up Cognitive Services/index.html b/docs/Get Started/Set up Cognitive Services/index.html
index 02dae8adb3..33a82fbe19 100644
--- a/docs/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Azure AI Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/en-us/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Azure AI Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create Azure AI service (Decision, Language, Speech, Vision) resource</a>. You can follow the steps at <a href="https://learn.microsoft.com/en-us/azure/ai-services/multi-service-resource?tabs=windows&amp;pivots=azportal#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">Create a multi-service resource for Azure AI services</a>. </li><li>Get your Azure AI Services resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret">Manage secrets with Azure Key Vault and access the secrets with find_secret<a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="hash-link" aria-label="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret" title="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret">​</a></h2><p>After you create an Azure AI resource, you will obtain a resource key. You can use this resource key directly in our notebooks as a string, but we recommend to manage secrets with Azure Key Vault. Azure Key Vault is a cloud-based service that allows you to store and manage cryptographic keys, certificates, and secrets used by cloud applications and services.</p><p>You can skip the following content if you want to use a secret string instead of Azure Key Vault. This is not recommended for production workloads.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-key-vault">Create Azure Key Vault<a href="#create-azure-key-vault" class="hash-link" aria-label="Direct link to Create Azure Key Vault" title="Direct link to Create Azure Key Vault">​</a></h3><p>Refer to <a href="https://learn.microsoft.com/azure/key-vault/general/quick-create-portal" target="_blank" rel="noopener noreferrer">this article</a> to create a key vault using Azure Portal</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-secret-to-azure-key-vault">Save secret to Azure Key Vault<a href="#save-secret-to-azure-key-vault" class="hash-link" aria-label="Direct link to Save secret to Azure Key Vault" title="Direct link to Save secret to Azure Key Vault">​</a></h3><ul><li>Go to Access control (IAM) and assign <strong>Key Vault Administrator Role</strong> to yourself.</li><li>Go to Secrets and choose <strong>+ Generate/Import</strong>, create a key with the secret value obtained from Azure AI service.</li><li>Choose <strong>Create</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-microsoft-fabric--power-bi">Use find_secret on Microsoft Fabric / Power BI<a href="#use-find_secret-on-microsoft-fabric--power-bi" class="hash-link" aria-label="Direct link to Use find_secret on Microsoft Fabric / Power BI" title="Direct link to Use find_secret on Microsoft Fabric / Power BI">​</a></h3><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Granted the Fabric or Power BI Service Account Get permissions in the Azure Key Vault.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-azure-databricks">Use find_secret on Azure Databricks<a href="#use-find_secret-on-azure-databricks" class="hash-link" aria-label="Direct link to Use find_secret on Azure Databricks" title="Direct link to Use find_secret on Azure Databricks">​</a></h3><ul><li>In the Azure Portal, find and select the Azure Key Vault Instance. Select the <strong>Access configuration</strong> tab under Settings. Set Permission model to Vault access policy.</li><li>On Databricks, go to <code>https://&lt;databricks-instance&gt;#secrets/createScope</code>. This URL is case sensitive.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_create_secret_scope.png" width="600" class="img_ev3q"><ul><li>Enter the name of the secret scope and choose desired Manage Principal.</li><li>Go to your Azure Key Vault -&gt; Properties and find the <strong>DNS Name</strong> (Vault URI on Azure Key Vault) and <strong>Resource ID</strong>. Enter the DNS Name and Resource ID on Databricks createScope page.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_find_resource_id.png" width="600" class="img_ev3q"><ul><li>Click the <strong>Create</strong> Button.</li></ul><p>Refer to <a href="https://learn.microsoft.com/azure/databricks/security/secrets/secret-scopes#--create-an-azure-key-vault-backed-secret-scope" target="_blank" rel="noopener noreferrer">this article</a> for more details.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-synapse">Use find_secret on Synapse<a href="#use-find_secret-on-synapse" class="hash-link" aria-label="Direct link to Use find_secret on Synapse" title="Direct link to Use find_secret on Synapse">​</a></h3><ul><li>Open the Synapse Studio and go to the Manage tab.</li><li>Under External connections, select Linked services.</li><li>To add a linked service, select New.</li><li>Select the Azure Key Vault tile from the list and select Continue.</li><li>Enter a linked service name and choose the key vault you want to connect to.</li><li>Select Create</li><li>Click Publish</li></ul><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Choose <strong>Key Vault Administrator</strong> in Role blade, select Next.</li><li>In Members blade, choose Assign access to <strong>Managed identity</strong>. Select members, choose the subscription your Synapse Workspace in. For Managed identity, select Synapse workspace, choose your workspace.</li><li>Select <strong>Review + assign</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="quick-test">Quick Test<a href="#quick-test" class="hash-link" aria-label="Direct link to Quick Test" title="Direct link to Quick Test">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own key vault name and api name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own AI service location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your result should looks like</p><table><thead><tr><th>language</th><th>text</th><th>entities</th></tr></thead><tbody><tr><td>en</td><td>Dr. Smith has a very modern medical office, and she has great staff.</td><td>[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td>en</td><td>I had a wonderful trip to Seattle last week.</td><td>[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li><li><a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="table-of-contents__link toc-highlight">Manage secrets with Azure Key Vault and access the secrets with find_secret</a><ul><li><a href="#create-azure-key-vault" class="table-of-contents__link toc-highlight">Create Azure Key Vault</a></li><li><a href="#save-secret-to-azure-key-vault" class="table-of-contents__link toc-highlight">Save secret to Azure Key Vault</a></li><li><a href="#use-find_secret-on-microsoft-fabric--power-bi" class="table-of-contents__link toc-highlight">Use find_secret on Microsoft Fabric / Power BI</a></li><li><a href="#use-find_secret-on-azure-databricks" class="table-of-contents__link toc-highlight">Use find_secret on Azure Databricks</a></li><li><a href="#use-find_secret-on-synapse" class="table-of-contents__link toc-highlight">Use find_secret on Synapse</a></li><li><a href="#quick-test" class="table-of-contents__link toc-highlight">Quick Test</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Overview/index.html b/docs/Overview/index.html
index 41a414e555..b27dabe89d 100644
--- a/docs/Overview/index.html
+++ b/docs/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/estimators/estimators_causal/index.html b/docs/Quick Examples/estimators/estimators_causal/index.html
index afe737c55a..27ca2a147a 100644
--- a/docs/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/estimators/estimators_cognitive/index.html b/docs/Quick Examples/estimators/estimators_cognitive/index.html
index 7d8985cbb3..cf656727c8 100644
--- a/docs/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/estimators/estimators_core/index.html b/docs/Quick Examples/estimators/estimators_core/index.html
index 6574eb4af0..bfc3447832 100644
--- a/docs/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/Quick Examples/estimators/estimators_lightgbm/index.html
index ff6adfbfac..3b6f63ec76 100644
--- a/docs/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/estimators/estimators_vw/index.html b/docs/Quick Examples/estimators/estimators_vw/index.html
index a4ad0c4055..cc31b5231d 100644
--- a/docs/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/transformers/transformers_cognitive/index.html b/docs/Quick Examples/transformers/transformers_cognitive/index.html
index 30a61195e5..5f83124543 100644
--- a/docs/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1, p._2, 1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .++(baseSeq.map(p =&gt; (p._1, p._2, 2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;timestamp&quot;, &quot;value&quot;, &quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i * imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/transformers/transformers_core/index.html b/docs/Quick Examples/transformers/transformers_core/index.html
index 22785bc005..76d83dc790 100644
--- a/docs/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/Quick Examples/transformers/transformers_deep_learning/index.html
index 8de4ce40e3..42eaf4b2ee 100644
--- a/docs/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/transformers/transformers_opencv/index.html b/docs/Quick Examples/transformers/transformers_opencv/index.html
index bc670afd53..c51ae7a918 100644
--- a/docs/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Quick Examples/transformers/transformers_vw/index.html b/docs/Quick Examples/transformers/transformers_vw/index.html
index f3724c831c..badff1c411 100644
--- a/docs/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/Contributor Guide/index.html b/docs/Reference/Contributor Guide/index.html
index 8ea57aa0b7..e1e38607ad 100644
--- a/docs/Reference/Contributor Guide/index.html	
+++ b/docs/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/Developer Setup/index.html b/docs/Reference/Developer Setup/index.html
index e08bffb359..d175edf2ab 100644
--- a/docs/Reference/Developer Setup/index.html	
+++ b/docs/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/Docker Setup/index.html b/docs/Reference/Docker Setup/index.html
index fbbaaaa04c..3fb8e0d089 100644
--- a/docs/Reference/Docker Setup/index.html	
+++ b/docs/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/Dotnet Setup/index.html b/docs/Reference/Dotnet Setup/index.html
index af155e98dc..26b8203f46 100644
--- a/docs/Reference/Dotnet Setup/index.html	
+++ b/docs/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The Azure AI services on spark aint bad|   en-US| null|[{positive, null, {0.99, 0.01, 0.00}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Reference/R Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">R setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - LightGBM in Dotnet</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/Reference/Quickstart - LightGBM in Dotnet/index.html
index b6a3304a00..3475f6a593 100644
--- a/docs/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Reference/Contributor Guide/">Contributor Guide</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Reference/Developer Setup/">Developer Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Reference/Docker Setup/">Docker Setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Reference/R Setup/">R setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Reference/Dotnet Setup/">.NET setup</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">Quickstart - LightGBM in Dotnet</a></li></ul></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Reference</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - LightGBM in Dotnet</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Reference/R Setup/index.html b/docs/Reference/R Setup/index.html
index 98a6afccdd..0e70f740e7 100644
--- a/docs/Reference/R Setup/index.html	
+++ b/docs/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Reference/Dotnet Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">.NET setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Use with MLFlow/Autologging/index.html b/docs/Use with MLFlow/Autologging/index.html
index 1707ded650..292bb99883 100644
--- a/docs/Use with MLFlow/Autologging/index.html	
+++ b/docs/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Use with MLFlow/Install/index.html b/docs/Use with MLFlow/Install/index.html
index 6d11d8bf6a..ac3eb39c9f 100644
--- a/docs/Use with MLFlow/Install/index.html	
+++ b/docs/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/Use with MLFlow/Overview/index.html b/docs/Use with MLFlow/Overview/index.html
index baa342ef37..885aecbc6f 100644
--- a/docs/Use with MLFlow/Overview/index.html	
+++ b/docs/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: 1.0.4</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Deploy Models/Overview/index.html b/docs/next/Deploy Models/Overview/index.html
index 15c595320b..f13c094448 100644
--- a/docs/next/Deploy Models/Overview/index.html	
+++ b/docs/next/Deploy Models/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -89,7 +89,7 @@
 It&#x27;s important to pick a gateway that has good connectivity to your spark cluster.
 For best performance and ease of configuration, we suggest using Spark Serving
 on an open cluster environment such as Kubernetes, Mesos, or Azure Batch.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="parameters">Parameters<a href="#parameters" class="hash-link" aria-label="Direct link to Parameters" title="Direct link to Parameters">​</a></h2><table><thead><tr><th>Parameter Name</th><th>Description</th><th>Necessary</th><th>Default Value</th><th>Applicable When</th></tr></thead><tbody><tr><td>host</td><td>The host to spin up a server on</td><td>Yes</td><td></td><td></td></tr><tr><td>port</td><td>The starting port when creating the web services.  Web services will increment this port several  times to find an open port.  In the future, the flexibility of this param will be expanded</td><td>yes</td><td></td><td></td></tr><tr><td>name</td><td>The Path of the api a user would call. The format is <code>hostname:port/name</code></td><td>yes</td><td></td><td></td></tr><tr><td>forwarding.enabled</td><td>Whether to forward the services to a gateway machine</td><td>no</td><td>false</td><td>When you need to forward services out of  a protected network. Only Supported for Continuous Serving.</td></tr><tr><td>forwarding.username</td><td>the username to connect to on the remote host</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.sshport</td><td>the port to ssh connect to</td><td>no</td><td>22</td><td></td></tr><tr><td>forwarding.sshHost</td><td>the host of the gateway machine</td><td>no</td><td></td><td></td></tr><tr><td>forwarding.keySas</td><td>A Secure access link that can be used to automatically download the required ssh private key</td><td>no</td><td></td><td>Sometimes more convenient than a directory</td></tr><tr><td>forwarding.keyDir</td><td>A directory on the machines holding the private key</td><td>no</td><td>&quot;~/.ssh&quot;</td><td>Useful if you can&#x27;t send keys over the wire securely</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Autologging</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#an-engine-for-deploying-spark-jobs-as-distributed-web-services" class="table-of-contents__link toc-highlight">An Engine for Deploying Spark Jobs as Distributed Web Services</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#jupyter-notebook-examples" class="table-of-contents__link toc-highlight">Jupyter Notebook Examples</a></li><li><a href="#spark-serving-hello-world" class="table-of-contents__link toc-highlight">Spark Serving Hello World</a></li><li><a href="#deploying-a-deep-network-with-the-cntkmodel" class="table-of-contents__link toc-highlight">Deploying a Deep Network with the CNTKModel</a></li></ul></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a><ul><li><a href="#head-node-load-balanced" class="table-of-contents__link toc-highlight">Head Node Load Balanced</a></li><li><a href="#fully-distributed-custom-load-balancer" class="table-of-contents__link toc-highlight">Fully Distributed (Custom Load Balancer)</a></li><li><a href="#sub-millisecond-latency-with-continuous-processing" class="table-of-contents__link toc-highlight">Sub-Millisecond Latency with Continuous Processing</a></li></ul></li><li><a href="#parameters" class="table-of-contents__link toc-highlight">Parameters</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Deploy Models/Quickstart - Deploying a Classifier/index.html b/docs/next/Deploy Models/Quickstart - Deploying a Classifier/index.html
index 03618cbaf1..807fba6b25 100644
--- a/docs/next/Deploy Models/Quickstart - Deploying a Classifier/index.html	
+++ b/docs/next/Deploy Models/Quickstart - Deploying a Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 tokenizes, hashes strings, one-hot encodes categorical variables, assembles the features into a vector
 etc.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we score it against the test dataset and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedClassifierModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we will define the webservice input/output.
 For more information, you can visit the <a href="https://github.com/Microsoft/SynapseML/blob/master/docs/mmlspark-serving.md" target="_blank" rel="noopener noreferrer">documentation for Spark Serving</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_inputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">readStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">address</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;localhost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8898</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parseRequest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">serving_outputs </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">serving_inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makeReply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serving_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeStream</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">replyTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">queryName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;my_query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;checkpointLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file:///tmp/checkpoints-{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">uuid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uuid1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Test the webservice</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; 10th&quot;,&quot;marital-status&quot;:&quot;Divorced&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;{&quot;education&quot;:&quot; Masters&quot;,&quot;marital-status&quot;:&quot;Married-civ-spouse&quot;,&quot;hours-per-week&quot;:40.0}&#x27;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://localhost:8898/my_api&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Response {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait for server to finish setting up (just to be safe)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">server</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contributor Guide</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#model-deployment-with-spark-serving" class="table-of-contents__link toc-highlight">Model Deployment with Spark Serving</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html b/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html
index 20aa61dcf5..a21d15da19 100644
--- a/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Advanced Usage - Async, Batching, and Multi-Key</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>AI Services Advanced Guide: Asynchrony, Batching, Multi-Key</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-imports-and-keys">Step 1: Imports and Keys<a href="#step-1-imports-and-keys" class="hash-link" aria-label="Direct link to Step 1: Imports and Keys" title="Direct link to Step 1: Imports and Keys">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc = &quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-basic-usage">Step 2: Basic Usage<a href="#step-2-basic-usage" class="hash-link" aria-label="Direct link to Step 2: Basic Usage" title="Direct link to Step 2: Basic Usage">​</a></h2><table><thead><tr><th align="center">Image 1</th><th align="center">Image 2</th><th align="center">Image 3</th></tr></thead><tbody><tr><td align="center">!<img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/objects.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/dog.jpg" width="300" class="img_ev3q"></td><td align="center"><img loading="lazy" src="https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/house.jpg" width="300" class="img_ev3q"></td></tr></tbody></table><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.vision import AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe with the image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url = &quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [(base_url + &quot;objects.jpg&quot;,), (base_url + &quot;dog.jpg&quot;,), (base_url + &quot;house.jpg&quot;,)],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;image&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Run the Computer Vision service. Analyze Image extracts infortmation from/about the images.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzer = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setVisualFeatures(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        [&quot;Categories&quot;, &quot;Color&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;Objects&quot;, &quot;Tags&quot;]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;analysis_results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setImageUrlCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_results = analyzer.transform(image_df).cache()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="first-well-look-at-the-full-response-objects">First we&#x27;ll look at the full response objects:<a href="#first-well-look-at-the-full-response-objects" class="hash-link" aria-label="Direct link to First we&#x27;ll look at the full response objects:" title="Direct link to First we&#x27;ll look at the full response objects:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="we-can-select-out-just-what-we-need">We can select out just what we need:<a href="#we-can-select-out-just-what-we-need" class="hash-link" aria-label="Direct link to We can select out just what we need:" title="Direct link to We can select out just what we need:">​</a></h4><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(image_results.select(&quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="whats-going-on-under-the-hood">What&#x27;s going on under the hood<a href="#whats-going-on-under-the-hood" class="hash-link" aria-label="Direct link to What&#x27;s going on under the hood" title="Direct link to What&#x27;s going on under the hood">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/cog_service.svg" width="300" class="img_ev3q"><p>When we call the AI service transformer, we start cognitive service clients on each of your spark workers.
 These clients send requests to the cloud, and turn the JSON responses into Spark Struct Types so that you can access any field that the service returns.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-asynchronous-usage">Step 3: Asynchronous Usage<a href="#step-3-asynchronous-usage" class="hash-link" aria-label="Direct link to Step 3: Asynchronous Usage" title="Direct link to Step 3: Asynchronous Usage">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_parallelism.svg" width="700" class="img_ev3q"><p>Apache Spark ordinarily parallelizes a computation to all of it&#x27;s worker threads. When working with services however this parallelism doesent fully maximize throughput because workers sit idle as requests are processed on the server. The <code>concurrency</code> parameter makes sure that each worker can stay busy as they wait for requests to complete.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(analyzer.setConcurrency(3).transform(image_df))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="faster-without-extra-hardware">Faster without extra hardware:<a href="#faster-without-extra-hardware" class="hash-link" aria-label="Direct link to Faster without extra hardware:" title="Direct link to Faster without extra hardware:">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/async_relative%20(2).png" width="500" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-batching">Step 4: Batching<a href="#step-4-batching" class="hash-link" aria-label="Direct link to Step 4: Batching" title="Direct link to Step 4: Batching">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.services.language import AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Create a dataframe</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_df = spark.createDataFrame(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am so happy today, its sunny!&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;I am frustrated by this rush hour traffic&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        (&quot;The AI services on spark is pretty lit&quot;,),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    [&quot;text&quot;],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKind(&quot;SentimentAnalysis&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLocation(service_loc)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSubscriptionKey(service_key)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sentiment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setErrorCol(&quot;error&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBatchSize(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"># Show the results of your text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(sentiment.transform(text_df).select(&quot;text&quot;, &quot;sentiment.documents.sentiment&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-multi-key">Step 5: Multi-Key<a href="#step-5-multi-key" class="hash-link" aria-label="Direct link to Step 5: Multi-Key" title="Direct link to Step 5: Multi-Key">​</a></h2><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key_2 = find_secret(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name=&quot;ai-services-api-key-2&quot;, keyvault=&quot;mmlspark-build-keys&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keys = [service_key, service_key_2]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">@udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def random_key():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    return keys[random.randint(0, len(keys) - 1)]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df2 = image_df.withColumn(&quot;key&quot;, random_key())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results = analyzer.setSubscriptionKeyCol(&quot;key&quot;).transform(image_df2)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(results.select(&quot;key&quot;, &quot;analysis_results.description.captions.text&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="learn-more">Learn More<a href="#learn-more" class="hash-link" aria-label="Direct link to Learn More" title="Direct link to Learn More">​</a></h2><ul><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Explore other cogntive services</a></li><li><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Read our paper &quot;Large-Scale Intelligent Microservices&quot;</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-imports-and-keys" class="table-of-contents__link toc-highlight">Step 1: Imports and Keys</a></li><li><a href="#step-2-basic-usage" class="table-of-contents__link toc-highlight">Step 2: Basic Usage</a></li><li><a href="#step-3-asynchronous-usage" class="table-of-contents__link toc-highlight">Step 3: Asynchronous Usage</a></li><li><a href="#step-4-batching" class="table-of-contents__link toc-highlight">Step 4: Batching</a></li><li><a href="#step-5-multi-key" class="table-of-contents__link toc-highlight">Step 5: Multi-Key</a></li><li><a href="#learn-more" class="table-of-contents__link toc-highlight">Learn More</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Geospatial Services/index.html b/docs/next/Explore Algorithms/AI Services/Geospatial Services/index.html
index 38a5eccdaf..fa575015f8 100644
--- a/docs/next/Explore Algorithms/AI Services/Geospatial Services/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Geospatial Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Geospatial Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Geospatial Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Geospatial Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Geospatial Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Geospatial Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Geospatial Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Geospatial Services</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="500" src="https://azurecomcdn.azureedge.net/cvt-18f087887a905ed3ae5310bee894aa53fc03cfffadc5dc9902bfe3469d832fec/less/images/section/azure-maps.png" class="img_ev3q"><h1>Azure Maps Geospatial Services</h1><p><a href="https://azure.microsoft.com/services/azure-maps/" target="_blank" rel="noopener noreferrer">Microsoft Azure Maps </a> provides developers from all industries with powerful geospatial capabilities. Those geospatial capabilities are packed with the freshest mapping data. Azure Maps is available for web, mobile (iOS and Android), Microsoft Power BI, Microsoft Power Apps and Microsoft Synapse. Azure Maps is an Open API compliant set of REST APIs. The following are only a high-level overview of the services which Azure Maps offers - Maps, Search, Routing, Traffic, Weather, Time Zones, Geolocation, Geofencing, Map Data, Creator, and Spatial Operations.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="geocode-addresses">Geocode addresses<a href="#geocode-addresses" class="hash-link" aria-label="Direct link to Geocode addresses" title="Direct link to Geocode addresses">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-batch" target="_blank" rel="noopener noreferrer"><strong>Address Geocoding</strong></a> The Search Address Batch API sends batches of queries to Search Address API using just a single API call. This API geocodes text addresses or partial addresses and the geocoding search index will be queried for everything above the street level data. <strong>Note</strong> that the geocoder is very tolerant of typos and incomplete addresses. It will also handle everything from exact street addresses or street or intersections as well as higher level geographies such as city centers, counties, states etc.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocode-coordinates">Reverse Geocode Coordinates<a href="#reverse-geocode-coordinates" class="hash-link" aria-label="Direct link to Reverse Geocode Coordinates" title="Direct link to Reverse Geocode Coordinates">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/search/post-search-address-reverse-batch" target="_blank" rel="noopener noreferrer"><strong>Reverse Geocoding</strong></a> The Search Address Reverse Batch API sends batches of queries to Search Address Reverse API using just a single API call. This API takes in location coordinates and translates them into human readable street addresses. Most often this is needed in tracking applications where you receive a GPS feed from the device or asset and wish to know what address where the coordinate is located.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="get-point-in-polygon">Get Point In Polygon<a href="#get-point-in-polygon" class="hash-link" aria-label="Direct link to Get Point In Polygon" title="Direct link to Get Point In Polygon">​</a></h3><p><a href="https://docs.microsoft.com/rest/api/maps/spatial/get-point-in-polygon" target="_blank" rel="noopener noreferrer"><strong>Get Point in Polygon</strong></a> This API returns a boolean value indicating whether a point is inside a set of polygons. The set of polygons can we pre-created by using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>  referenced by a unique udid.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Sign into the <a href="https://portal.azure.com" target="_blank" rel="noopener noreferrer">Azure Portal</a> and create an Azure Maps account by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-account-keys#create-a-new-account" target="_blank" rel="noopener noreferrer">instructions</a>.</li><li>Once the Maps account is created, provision a Maps Creator Resource by following these <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-creator#create-creator-resource" target="_blank" rel="noopener noreferrer">instructions</a>. Creator is a <a href="https://docs.microsoft.com/azure/azure-maps/creator-geographic-scope" target="_blank" rel="noopener noreferrer">geographically scoped service</a>. Pick appropriate location while provisioning the creator resource. </li><li>Follow these <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/overview-cognitive-services#create-an-apache-spark-cluster" target="_blank" rel="noopener noreferrer">instructions</a> to set up your Azure Databricks environment and install SynapseML.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace the <code>AZUREMAPS_API_KEY</code> placeholders with your own <a href="https://docs.microsoft.com/azure/azure-maps/how-to-manage-authentication#view-authentication-details" target="_blank" rel="noopener noreferrer">Maps account key</a>.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="shared-code">Shared code<a href="#shared-code" class="hash-link" aria-label="Direct link to Shared code" title="Direct link to Shared code">​</a></h2><p>To get started, we&#x27;ll need to add this code to the project:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="geocoding-sample">Geocoding sample<a href="#geocoding-sample" class="hash-link" aria-label="Direct link to Geocoding sample" title="Direct link to Geocoding sample">​</a></h2><p>The azure maps geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/getsearchaddress" target="_blank" rel="noopener noreferrer">Search Address API</a>. The API limits the batch size to 10000 queries per request.  </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;One, Microsoft Way, Redmond&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;400 Broad St, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;350 5th Ave, New York&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pike Pl, Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Champ de Mars, 5 Avenue Anatole France, 75007 Paris&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="reverse-geocoding-sample">Reverse Geocoding sample<a href="#reverse-geocoding-sample" class="hash-link" aria-label="Direct link to Reverse Geocoding sample" title="Direct link to Reverse Geocoding sample">​</a></h2><p>The azure maps reverse geocoder sends batches of queries to the <a href="https://docs.microsoft.com/rest/api/maps/search/get-search-address-reverse" target="_blank" rel="noopener noreferrer">Search Address Reverse API</a> using just a single API call. The API allows caller to batch up to 10,000 queries per request</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rev_geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ReverseAddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rev_geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;freeformAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.addresses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;address&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="check-point-in-polygon-sample">Check Point In Polygon sample<a href="#check-point-in-polygon-sample" class="hash-link" aria-label="Direct link to Check Point In Polygon sample" title="Direct link to Check Point In Polygon sample">​</a></h2><p>This API returns a boolean value indicating whether a point is inside a set of polygons. The polygon can be added to your creator account using the <a href="https://docs.microsoft.com/rest/api/maps/data/upload-preview" target="_blank" rel="noopener noreferrer"><strong>Data Upload API</strong></a>. The API then returns a unique udid to reference the polygon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-geojson-polygons-in-your-azure-maps-creator-account">Set up geojson Polygons in your azure maps creator account<a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="hash-link" aria-label="Direct link to Set up geojson Polygons in your azure maps creator account" title="Direct link to Set up geojson Polygons in your azure maps creator account">​</a></h3><p>Based on where the creator resource was provisioned, we need to prefix the appropriate geography code to the azure maps URL. In this example, the assumption is that the creator resource was provisioned in <code>East US 2</code> Location and hence we pick <code>us</code> as our geo prefix. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a geography, you want your data to reside in.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Allowed values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># us =&gt; North American datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># eu -&gt; European datacenters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">url_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload a geojson with polygons in them</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureCollection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;properties&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometryId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test_geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;geometry&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;coordinates&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.03956604003906</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.7483271435476</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.14290618896484</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">47.67856488312544</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">long_running_operation </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Sometimes this may take upto 30 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Status Code: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">r</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">status_code</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, Long Running Operation: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This Operation completes in approximately 5 ~ 15 seconds</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">long_running_operation</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-function-to-check-if-point-is-in-polygon">Use the function to check if point is in polygon<a href="#use-the-function-to-check-if-point-is-in-polygon" class="hash-link" aria-label="Direct link to Use the function to check if point is in polygon" title="Direct link to Use the function to check if point is in polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">48.858561</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.294911</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.639765</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.127896</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.621028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.348170</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">47.734012</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">122.102737</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Azure Maps geocoder to enhance the data with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup">Cleanup<a href="#cleanup" class="hash-link" aria-label="Direct link to Cleanup" title="Direct link to Cleanup">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">url_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multivariate Anomaly Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#geocode-addresses" class="table-of-contents__link toc-highlight">Geocode addresses</a></li><li><a href="#reverse-geocode-coordinates" class="table-of-contents__link toc-highlight">Reverse Geocode Coordinates</a></li><li><a href="#get-point-in-polygon" class="table-of-contents__link toc-highlight">Get Point In Polygon</a></li></ul></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#shared-code" class="table-of-contents__link toc-highlight">Shared code</a></li><li><a href="#geocoding-sample" class="table-of-contents__link toc-highlight">Geocoding sample</a></li><li><a href="#reverse-geocoding-sample" class="table-of-contents__link toc-highlight">Reverse Geocoding sample</a></li><li><a href="#check-point-in-polygon-sample" class="table-of-contents__link toc-highlight">Check Point In Polygon sample</a><ul><li><a href="#set-up-geojson-polygons-in-your-azure-maps-creator-account" class="table-of-contents__link toc-highlight">Set up geojson Polygons in your azure maps creator account</a></li><li><a href="#use-the-function-to-check-if-point-is-in-polygon" class="table-of-contents__link toc-highlight">Use the function to check if point is in polygon</a></li><li><a href="#cleanup" class="table-of-contents__link toc-highlight">Cleanup</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html b/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html
index 0b0149633c..abdf17fb5d 100644
--- a/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multivariate Anomaly Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Azure AI Services - Multivariate Anomaly Detection</h1><p>This recipe shows how you can use SynapseML and Azure AI services on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train a model for multivariate anomaly detection using the Azure AI services, and we then use to the model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Azure AI Anomaly Detector, refer to <a href="https://docs.microsoft.com/azure/ai-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">this documentation page</a>. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Follow the instructions to create an <code>Anomaly Detector</code> resource using the Azure portal or alternatively, you can also use the Azure CLI to create this resource.</p><ul><li>In the Azure portal, select <strong>Create</strong> in your resource group, and then type <strong>Anomaly Detector</strong>. Select the Anomaly Detector resource.</li><li>Give the resource a name, and ideally use the same region as the rest of your resource group. Use the default options for the rest, and then select <strong>Review + Create</strong> and then <strong>Create</strong>.</li><li>Once the Anomaly Detector resource is created, open it and select the <code>Keys and Endpoints</code> panel in the left nav. Copy the key for the Anomaly Detector resource into the <code>ANOMALY_API_KEY</code> environment variable, or store it in the <code>anomalyKey</code> variable.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-storage-account-resource">Create a Storage Account resource<a href="#create-a-storage-account-resource" class="hash-link" aria-label="Direct link to Create a Storage Account resource" title="Direct link to Create a Storage Account resource">​</a></h3><p>In order to save intermediate data, you need to create an Azure Blob Storage Account. Within that storage account, create a container for storing the intermediate data. Make note of the container name, and copy the connection string to that container. You need it later to populate the <code>containerName</code> variable and the <code>BLOB_CONNECTION_STRING</code> environment variable.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h3><p>Let&#x27;s start by setting up the environment variables for our service keys. The next cell sets the <code>ANOMALY_API_KEY</code> and the <code>BLOB_CONNECTION_STRING</code> environment variables based on the values stored in our Azure Key Vault. If you&#x27;re running this tutorial in your own environment, make sure you set these environment variables before you proceed.</p><p>Now, lets read the <code>ANOMALY_API_KEY</code> and <code>BLOB_CONNECTION_STRING</code> environment variables and set the <code>containerName</code> and <code>location</code> variables.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Dectector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own anomaly api key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Your storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage account name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A connection string to your blob storage account</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storageKey </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own storage key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A place to save intermediate MVAD results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The location of the anomaly detector resource that you created</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we connect to our storage account so that anomaly detector can save intermediate results there:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storageName</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storageKey</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s import all the necessary modules.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now, let&#x27;s read our sample data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/MVAD/sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Let&#x27;s inspect the dataframe:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now create an <code>estimator</code> object, which is used to train our model. We specify the start and end times for the training data. We also specify the input columns to use, and the name of the column that contains the timestamps. Finally, we specify the number of data points to use in the anomaly detection sliding window, and we set the connection string to the Azure Blob Storage Account. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-06-01T12:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T17:55:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we created the <code>estimator</code>, let&#x27;s fit it to the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once the training is done, we can now use the model for inference. The code in the next cell specifies the start and end times for the data we would like to detect the anomalies in. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-02T18:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2020-07-06T05:15:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When we called <code>.show(5)</code> in the previous cell, it showed us the first five rows in the dataframe. The results were all <code>null</code> because they weren&#x27;t inside the inference window.</p><p>To show the results only for the inferred data, lets select the columns we need. We can then order the rows in the dataframe by ascending order, and filter the result to only show the rows that are in the range of the inference window. In our case <code>inferenceEndTime</code> is the same as the last row in the dataframe, so can ignore that. </p><p>Finally, to be able to better plot the results, lets convert the Spark dataframe to a Pandas dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results.severity&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Format the <code>contributors</code> column that stores the contribution score from each sensor to the detected anomalies. The next cell formats this data, and splits the contribution score of each sensor into its own column.</p><p>For Spark3.3 and below versions, the output of select statements will be in the format of <code>List&lt;Rows&gt;</code>, so to format the data into dictionary and generate the values when interpretation is empty, please use the below parse method:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def parse(x):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if len(x) &gt; 0:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return dict([item[:2] for item in x])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        return {&quot;sensor_1&quot;: 0, &quot;sensor_2&quot;: 0, &quot;sensor_3&quot;: 0}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Staring with Spark3.4, the output of the select statement is already formatted as a <code>numpy.ndarry&lt;dictionary&gt;</code> and no need to format the data again, so please use below parse method to generate the values when interpretation is empty:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;interpretation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json_normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;contributors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rdf</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Great! We now have the contribution scores of sensors 1, 2, and 3 in the <code>series_0</code>, <code>series_1</code>, and <code>series_2</code> columns respectively. </p><p>Run the next cell to plot the results. The <code>minSeverity</code> parameter specifies the minimum severity of the anomalies to be plotted.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">minSeverity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Main Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;A plot of the values from the three sensors with the detected anomalies highlighted in red.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Severity Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">minSeverity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;severity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;minSeverity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Severity of the detected anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">####### Contributors Figure #######</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The contribution of each sensor to the detected anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" width="1300" src="https://mmlspark.blob.core.windows.net/graphics/multivariate-anomaly-detection-plot.png" class="img_ev3q"><p>The plots show the raw data from the sensors (inside the inference window) in orange, green, and blue. The red vertical lines in the first figure show the detected anomalies that have a severity greater than or equal to <code>minSeverity</code>. </p><p>The second plot shows the severity score of all the detected anomalies, with the <code>minSeverity</code> threshold shown in the dotted red line.</p><p>Finally, the last plot shows the contribution of the data from each sensor to the detected anomalies. It helps us diagnose and understand the most likely cause of each anomaly.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Geospatial Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li><li><a href="#create-a-storage-account-resource" class="table-of-contents__link toc-highlight">Create a Storage Account resource</a></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Overview/index.html b/docs/next/Explore Algorithms/AI Services/Overview/index.html
index e5dca9d768..d6ae43f81a 100644
--- a/docs/next/Explore Algorithms/AI Services/Overview/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure AI services</h1><image width="200" alt-text="icon" src="https://mmlspark.blob.core.windows.net/graphics/Readme/cog_services_on_spark_2.svg"></image><p>Azure AI services help developers and organizations rapidly create intelligent, cutting-edge, market-ready, and responsible applications with out-of-the-box and pre-built and customizable APIs and models.</p><p>SynapseML allows you to build powerful and highly scalable predictive and analytical models from various Spark data sources. Synapse Spark provide built-in SynapseML libraries including synapse.ml.services.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites-on-azure-databricks">Prerequisites on Azure Databricks<a href="#prerequisites-on-azure-databricks" class="hash-link" aria-label="Direct link to Prerequisites on Azure Databricks" title="Direct link to Prerequisites on Azure Databricks">​</a></h2><ol><li>Follow the steps in <a href="https://docs.microsoft.com/azure/services-services/big-data/getting-started" target="_blank" rel="noopener noreferrer">Getting started</a> to set up your Azure Databricks and Azure AI services environment. This tutorial shows you how to install SynapseML and how to create your Spark cluster in Databricks.</li><li>After you create a new notebook in Azure Databricks, copy the <strong>Shared code</strong> below and paste into a new cell in your notebook.</li><li>Choose a service sample, below, and copy paste it into a second new cell in your notebook.</li><li>Replace any of the service subscription key placeholders with your own key.</li><li>Choose the run button (triangle icon) in the upper right corner of the cell, then select <strong>Run Cell</strong>.</li><li>View results in a table below the cell.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites-on-azure-synapse-analytics">Prerequisites on Azure Synapse Analytics<a href="#prerequisites-on-azure-synapse-analytics" class="hash-link" aria-label="Direct link to Prerequisites on Azure Synapse Analytics" title="Direct link to Prerequisites on Azure Synapse Analytics">​</a></h2><p>The tutorial, <a href="https://learn.microsoft.com/azure/synapse-analytics/machine-learning/tutorial-configure-cognitive-services-synapse" target="_blank" rel="noopener noreferrer">Pre-requisites for using Azure AI services in Azure Synapse</a>, walks you through a couple steps you need to perform before using Azure AI services in Synapse Analytics.</p><p><a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI services</a> is a suite of APIs, SDKs, and services that developers can use to add intelligent features to their applications. AI services empower developers even when they don&#x27;t have direct AI or data science skills or knowledge. Azure AI services help developers create applications that can see, hear, speak, understand, and even begin to reason. The catalog of services within Azure AI services can be categorized into five main pillars: Vision, Speech, Language, Web search, and Decision.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vision">Vision<a href="#vision" class="hash-link" aria-label="Direct link to Vision" title="Direct link to Vision">​</a></h3><p><a href="https://azure.microsoft.com/services/cognitive-services/computer-vision/" target="_blank" rel="noopener noreferrer"><strong>Computer Vision</strong></a></p><ul><li>Describe: provides description of an image in human readable language (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/DescribeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.DescribeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze (color, image type, face, adult/racy content): analyzes visual features of an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/AnalyzeImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.AnalyzeImage" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>OCR: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/OCR.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.OCR" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize Text: reads text from an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Thumbnail: generates a thumbnail of user-specified size from the image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/GenerateThumbnails.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.GenerateThumbnails" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Recognize domain-specific content: recognizes domain-specific content (celebrity, landmark) (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/RecognizeDomainSpecificContent.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.RecognizeDomainSpecificContent" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Tag: identifies list of words that are relevant to the input image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/vision/TagImage.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.vision.html#module-synapse.ml.services.vision.TagImage" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><p><a href="https://azure.microsoft.com/services/cognitive-services/face/" target="_blank" rel="noopener noreferrer"><strong>Face</strong></a></p><ul><li>Detect: detects human faces in an image (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/DetectFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.DetectFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Verify: verifies whether two faces belong to a same person, or a face belongs to a person (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/VerifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.VerifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Identify: finds the closest matches of the specific query person face from a person group (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/IdentifyFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.IdentifyFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find similar: finds similar faces to the query face in a face list (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/FindSimilarFace.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.FindSimilarFace" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Group: divides a group of faces into disjoint groups based on similarity (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/face/GroupFaces.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.face.html#module-synapse.ml.services.face.GroupFaces" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speech">Speech<a href="#speech" class="hash-link" aria-label="Direct link to Speech" title="Direct link to Speech">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-speech" target="_blank" rel="noopener noreferrer"><strong>Speech Services</strong></a></p><ul><li>Speech-to-text: transcribes audio streams (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/SpeechToText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.SpeechToText" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Conversation Transcription: transcribes audio streams into live transcripts with identified speakers. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/ConversationTranscription.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.ConversationTranscription" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Text to Speech: Converts text to realistic audio (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/speech/TextToSpeech.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.speech.html#module-synapse.ml.services.speech.TextToSpeech" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="language">Language<a href="#language" class="hash-link" aria-label="Direct link to Language" title="Direct link to Language">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-language" target="_blank" rel="noopener noreferrer"><strong>AI Language</strong></a></p><ul><li>Language detection: detects language of the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/LanguageDetector.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.LanguageDetector" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Key phrase extraction: identifies the key talking points in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/KeyPhraseExtractor.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.KeyPhraseExtractor" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Named entity recognition: identifies known entities and general named entities in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/NER.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.NER" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Sentiment analysis: returns a score between 0 and 1 indicating the sentiment in the input text (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/TextSentiment.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.TextSentiment" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Healthcare Entity Extraction: Extracts medical entities and relationships from text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/text/AnalyzeHealthText.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.text.html#module-synapse.ml.services.text.AnalyzeHealthText" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translation">Translation<a href="#translation" class="hash-link" aria-label="Direct link to Translation" title="Direct link to Translation">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/translator" target="_blank" rel="noopener noreferrer"><strong>Translator</strong></a></p><ul><li>Translate: Translates text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Translate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Translate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Transliterate: Converts text in one language from one script to another script. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Transliterate.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Transliterate" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Detect: Identifies the language of a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/Detect.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.Detect" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>BreakSentence: Identifies the positioning of sentence boundaries in a piece of text. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/BreakSentence.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.BreakSentence" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Lookup: Provides alternative translations for a word and a small number of idiomatic phrases. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryLookup.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryLookup" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Dictionary Examples: Provides examples that show how terms in the dictionary are used in context. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DictionaryExamples.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DictionaryExamples" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Document Translation: Translates documents across all supported languages and dialects while preserving document structure and data format. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/translate/DocumentTranslator.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.translate.html#module-synapse.ml.services.translate.DocumentTranslator" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="document-intelligence">Document Intelligence<a href="#document-intelligence" class="hash-link" aria-label="Direct link to Document Intelligence" title="Direct link to Document Intelligence">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence/" target="_blank" rel="noopener noreferrer"><strong>Document Intelligence</strong></a></p><ul><li>Analyze Layout: Extract text and layout information from a given document. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeLayout.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeLayout" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Receipts: Detects and extracts data from receipts using optical character recognition (OCR) and our receipt model, enabling you to easily extract structured data from receipts such as merchant name, merchant phone number, transaction date, transaction total, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeReceipts.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeReceipts" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Business Cards: Detects and extracts data from business cards using optical character recognition (OCR) and our business card model, enabling you to easily extract structured data from business cards such as contact names, company names, phone numbers, emails, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeBusinessCards.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeBusinessCards" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Invoices: Detects and extracts data from invoices using optical character recognition (OCR) and our invoice understanding deep learning models, enabling you to easily extract structured data from invoices such as customer, vendor, invoice ID, invoice due date, total, invoice amount due, tax amount, ship to, bill to, line items and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeInvoices.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeInvoices" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze ID Documents: Detects and extracts data from identification documents using optical character recognition (OCR) and our ID document model, enabling you to easily extract structured data from ID documents such as first name, last name, date of birth, document number, and more. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeIDDocuments.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeIDDocuments" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Analyze Custom Form: Extracts information from forms (PDFs and images) into structured data based on a model created from a set of representative training forms. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/AnalyzeCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.AnalyzeCustomModel" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Get Custom Model: Get detailed information about a custom model. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/GetCustomModel.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>List Custom Models: Get information about all custom models. (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/form/ListCustomModels.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.ListCustomModels" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="decision">Decision<a href="#decision" class="hash-link" aria-label="Direct link to Decision" title="Direct link to Decision">​</a></h3><p><a href="https://azure.microsoft.com/products/ai-services/ai-anomaly-detector" target="_blank" rel="noopener noreferrer"><strong>Anomaly Detector</strong></a></p><ul><li>Anomaly status of latest point: generates a model using preceding points and determines whether the latest point is anomalous (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectLastAnomaly.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectLastAnomaly" target="_blank" rel="noopener noreferrer">Python</a>)</li><li>Find anomalies: generates a model using an entire series and finds anomalies in the series (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/anomaly/DetectAnomalies.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.anomaly.html#module-synapse.ml.services.anomaly.DetectAnomalies" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="search">Search<a href="#search" class="hash-link" aria-label="Direct link to Search" title="Direct link to Search">​</a></h3><ul><li><a href="https://azure.microsoft.com/services/services-services/bing-image-search-api/" target="_blank" rel="noopener noreferrer"><strong>Bing Image search</strong></a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/bing/BingImageSearch.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.bing.html#module-synapse.ml.services.bing.BingImageSearch" target="_blank" rel="noopener noreferrer">Python</a>)</li><li><a href="https://docs.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer"><strong>Azure Cognitive search</strong></a> (<a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/services/search/AzureSearchWriter$.html" target="_blank" rel="noopener noreferrer">Scala</a>, <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.search.html#module-synapse.ml.services.search.AzureSearchWriter" target="_blank" rel="noopener noreferrer">Python</a>)</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-your-system">Prepare your system<a href="#prepare-your-system" class="hash-link" aria-label="Direct link to Prepare your system" title="Direct link to Prepare your system">​</a></h2><p>To begin, import required libraries and initialize your Spark session.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Import Azure AI services libraries and replace the keys and locations in the following code snippet with your Azure AI services key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A general AI services key for AI Language, Computer Vision and Document Intelligence (or use separate keys that belong to each service)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. service_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Bing Search v7 subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Anomaly Detector subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomaly_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A Translator subscription key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># An Azure search key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-sentiment-analysis-on-text">Perform sentiment analysis on text<a href="#perform-sentiment-analysis-on-text" class="hash-link" aria-label="Direct link to Perform sentiment analysis on text" title="Direct link to Perform sentiment analysis on text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-language/" target="_blank" rel="noopener noreferrer">AI Language</a> service provides several algorithms for extracting intelligent insights from text. For example, we can find the sentiment of given input text. The service will return a score between 0.0 and 1.0 where low scores indicate negative sentiment and high score indicates positive sentiment.  This sample uses three simple sentences and returns the sentiment for each.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe that&#x27;s tied to it&#x27;s column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The AI services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Text Analytics service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your text query in a table format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-text-analytics-for-health-data">Perform text analytics for health data<a href="#perform-text-analytics-for-health-data" class="hash-link" aria-label="Direct link to Perform text analytics for health data" title="Direct link to Perform text analytics for health data">​</a></h2><p>The <a href="https://docs.microsoft.com/azure/ai-services/language-service/text-analytics-for-health/overview?tabs=ner" target="_blank" rel="noopener noreferrer">Text Analytics for Health Service</a> extracts and labels relevant medical information from unstructured text such as doctor&#x27;s notes, discharge summaries, clinical documents, and electronic health records.</p><p>The following code sample analyzes and transforms text from doctors notes into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;20mg of ibuprofen twice a day&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1tsp of Tylenol every 4 hours&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;6-drops of Vitamin B-12 every evening&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">healthcare </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeHealthText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">healthcare</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translate-text-into-a-different-language">Translate text into a different language<a href="#translate-text-into-a-different-language" class="hash-link" aria-label="Direct link to Translate text into a different language" title="Direct link to Translate text into a different language">​</a></h2><p><a href="https://azure.microsoft.com/services/ai-services/translator/" target="_blank" rel="noopener noreferrer">Translator</a> is a cloud-based machine translation service and is part of the Azure AI services family of AI APIs used to build intelligent apps. Translator is easy to integrate in your applications, websites, tools, and solutions. It allows you to add multi-language user experiences in 90 languages and dialects and can be used to translate text without hosting your own algorithm.</p><p>The following code sample does a simple text translation by providing the sentences you want to translate and target languages you want to translate them to.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe including sentences you want to translate</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Translator service with options</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="extract-information-from-a-document-into-structured-data">Extract information from a document into structured data<a href="#extract-information-from-a-document-into-structured-data" class="hash-link" aria-label="Direct link to Extract information from a document into structured data" title="Direct link to Extract information from a document into structured data">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence/" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> is a part of Azure Applied AI Services that lets you build automated data processing software using machine learning technology. With Azure AI Document Intelligence, you can identify and extract text, key/value pairs, selection marks, tables, and structure from your documents. The service outputs structured data that includes the relationships in the original file, bounding boxes, confidence and more.</p><p>The following code sample analyzes a business card image and extracts its information into structured data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe containing the source files</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Form Recognizer service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of recognition.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision-sample">Computer Vision sample<a href="#computer-vision-sample" class="hash-link" aria-label="Direct link to Computer Vision sample" title="Direct link to Computer Vision sample">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/ai-vision/" target="_blank" rel="noopener noreferrer">Azure AI Vision</a> analyzes images to identify structure such as faces, objects, and natural-language descriptions.</p><p>The following code sample analyzes images and labels them with <em>tags</em>. Tags are one-word descriptions of things in the image, such as recognizable objects, people, scenery, and actions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">base_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://raw.githubusercontent.com/Azure-Samples/cognitive-services-sample-data-files/master/ComputerVision/Images/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;objects.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dog.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">base_url </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;house.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Computer Vision service. Analyze Image extracts information from/about the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of what you wanted to pull out of the images.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analysis_results.description.tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="search-for-images-that-are-related-to-a-natural-language-query">Search for images that are related to a natural language query<a href="#search-for-images-that-are-related-to-a-natural-language-query" class="hash-link" aria-label="Direct link to Search for images that are related to a natural language query" title="Direct link to Search for images that are related to a natural language query">​</a></h2><p><a href="https://www.microsoft.com/bing/apis/bing-image-search-api" target="_blank" rel="noopener noreferrer">Bing Image Search</a> searches the web to retrieve images related to a user&#x27;s natural language query. </p><p>The following code sample uses a text query that looks for images with quotes. The output of the code is a list of image URLs that contain photos related to the query.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned, uncomment to use</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display(bingSearch.transform(bingParameters))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-speech-to-text">Transform speech to text<a href="#transform-speech-to-text" class="hash-link" aria-label="Direct link to Transform speech to text" title="Direct link to Transform speech to text">​</a></h2><p>The <a href="https://azure.microsoft.com/products/ai-services/ai-speech/" target="_blank" rel="noopener noreferrer">Speech-to-text</a> service converts streams or files of spoken audio to text. The following code sample transcribes one audio file to text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with our audio URLs, tied to the column called &quot;url&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Speech-to-text service to translate the audio into text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of the translation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text.DisplayText&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-text-to-speech">Transform text to speech<a href="#transform-text-to-speech" class="hash-link" aria-label="Direct link to Transform text to speech" title="Direct link to Transform text to speech">​</a></h2><p><a href="https://azure.microsoft.com/products/ai-services/text-to-speech/" target="_blank" rel="noopener noreferrer">Text to speech</a> is a service that allows you to build apps and services that speak naturally, choosing from more than 270 neural voices across 119 languages and variants.</p><p>The following code sample transforms text into an audio file that contains the content of the text.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with text and an output file location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Reading out loud is fun! Check out aka.ms/spark for more information&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            fs </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/output.mp3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-JennyNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_file&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check to make sure there were no errors during audio creation</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="detect-anomalies-in-time-series-data">Detect anomalies in time series data<a href="#detect-anomalies-in-time-series-data" class="hash-link" aria-label="Direct link to Detect anomalies in time series data" title="Direct link to Detect anomalies in time series data">​</a></h2><p>If you don&#x27;t have an anomaly detection resource created before Sep 20th 2023, you won&#x27;t be able to create one. You may want to skip this part.</p><p><a href="https://azure.microsoft.com/services/cognitive-services/anomaly-detector/" target="_blank" rel="noopener noreferrer">Anomaly Detector</a> is great for detecting irregularities in your time series data. The following code sample uses the Anomaly Detector service to find anomalies in a time series.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with the point data that Anomaly Detector requires</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;series1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Anomaly Detector service to look for irregular data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anamoly_detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomaly_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValueCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the full results of the analysis with the anomalies marked as &quot;True&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anamoly_detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-information-from-arbitrary-web-apis">Get information from arbitrary web APIs<a href="#get-information-from-arbitrary-web-apis" class="hash-link" aria-label="Direct link to Get information from arbitrary web APIs" title="Direct link to Get information from arbitrary web APIs">​</a></h2><p>With HTTP on Spark, any web service can be used in your big data pipeline. In this example, we use the <a href="http://api.worldbank.org/v2/country/" target="_blank" rel="noopener noreferrer">World Bank API</a> to get information about various countries around the world.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use any requests from the python requests library</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a dataframe with specifies which countries we want data on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Much faster for big data because of the concurrency :)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">client </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get the body of the response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> resp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">decode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the details of the country data returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    client</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">get_response_body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-search-sample">Azure AI search sample<a href="#azure-ai-search-sample" class="hash-link" aria-label="Direct link to Azure AI search sample" title="Direct link to Azure AI search sample">​</a></h2><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-33467690&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;analyzed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Geospatial Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#prerequisites-on-azure-databricks" class="table-of-contents__link toc-highlight">Prerequisites on Azure Databricks</a></li><li><a href="#prerequisites-on-azure-synapse-analytics" class="table-of-contents__link toc-highlight">Prerequisites on Azure Synapse Analytics</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#vision" class="table-of-contents__link toc-highlight">Vision</a></li><li><a href="#speech" class="table-of-contents__link toc-highlight">Speech</a></li><li><a href="#language" class="table-of-contents__link toc-highlight">Language</a></li><li><a href="#translation" class="table-of-contents__link toc-highlight">Translation</a></li><li><a href="#document-intelligence" class="table-of-contents__link toc-highlight">Document Intelligence</a></li><li><a href="#decision" class="table-of-contents__link toc-highlight">Decision</a></li><li><a href="#search" class="table-of-contents__link toc-highlight">Search</a></li></ul></li><li><a href="#prepare-your-system" class="table-of-contents__link toc-highlight">Prepare your system</a></li><li><a href="#perform-sentiment-analysis-on-text" class="table-of-contents__link toc-highlight">Perform sentiment analysis on text</a></li><li><a href="#perform-text-analytics-for-health-data" class="table-of-contents__link toc-highlight">Perform text analytics for health data</a></li><li><a href="#translate-text-into-a-different-language" class="table-of-contents__link toc-highlight">Translate text into a different language</a></li><li><a href="#extract-information-from-a-document-into-structured-data" class="table-of-contents__link toc-highlight">Extract information from a document into structured data</a></li><li><a href="#computer-vision-sample" class="table-of-contents__link toc-highlight">Computer Vision sample</a></li><li><a href="#search-for-images-that-are-related-to-a-natural-language-query" class="table-of-contents__link toc-highlight">Search for images that are related to a natural language query</a></li><li><a href="#transform-speech-to-text" class="table-of-contents__link toc-highlight">Transform speech to text</a></li><li><a href="#transform-text-to-speech" class="table-of-contents__link toc-highlight">Transform text to speech</a></li><li><a href="#detect-anomalies-in-time-series-data" class="table-of-contents__link toc-highlight">Detect anomalies in time series data</a></li><li><a href="#get-information-from-arbitrary-web-apis" class="table-of-contents__link toc-highlight">Get information from arbitrary web APIs</a></li><li><a href="#azure-ai-search-sample" class="table-of-contents__link toc-highlight">Azure AI search sample</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html
index 48734a0072..e2d49fee9f 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Celebrity Quotes</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Celebrity Quote Analysis with The Azure AI Services</h1><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> SQLTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># put your service keys here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="extracting-celebrity-quote-images-using-bing-image-search-on-spark">Extracting celebrity quote images using Bing Image Search on Spark<a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="hash-link" aria-label="Direct link to Extracting celebrity quote images using Bing Image Search on Spark" title="Direct link to Extracting celebrity quote images using Bing Image Search on Spark">​</a></h3><p>Here we define two Transformers to extract celebrity quote images.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%201.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># the number of images Bing will return for each query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i </span><span class="token operator">*</span><span class="token plain"> imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrity quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer to that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizing-images-of-celebrities">Recognizing Images of Celebrities<a href="#recognizing-images-of-celebrities" class="hash-link" aria-label="Direct link to Recognizing Images of Celebrities" title="Direct link to Recognizing Images of Celebrities">​</a></h3><p>This block identifies the name of the celebrities for each of the images returned by the Bing Image Search.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%202.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the first celebrity we see from the structured response</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">firstCeleb </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, celebs.result.celebrities[0].name as firstCeleb FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reading-the-quote-from-the-image">Reading the quote from the image.<a href="#reading-the-quote-from-the-image" class="hash-link" aria-label="Direct link to Reading the quote from the image." title="Direct link to Reading the quote from the image.">​</a></h3><p>This stage performs OCR on the images to recognize the quotes.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step%203.png" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recognizeText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> ocrRow </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">for</span><span class="token plain"> line </span><span class="token keyword">in</span><span class="token plain"> ocrRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recognitionResult</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># this transformer wil extract a simpler string from the structured output of recognize text</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getText </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getTextFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="understanding-the-sentiment-of-the-quote">Understanding the Sentiment of the Quote<a href="#understanding-the-sentiment-of-the-quote" class="hash-link" aria-label="Direct link to Understanding the Sentiment of the Quote" title="Direct link to Understanding the Sentiment of the Quote">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/step4.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentimentTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Extract the sentiment score from the API response body</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getSentiment </span><span class="token operator">=</span><span class="token plain"> SQLTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    statement</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SELECT *, sentiment.document.sentiment as sentimentLabel FROM __THIS__&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tying-it-all-together">Tying it all together<a href="#tying-it-all-together" class="hash-link" aria-label="Direct link to Tying it all together" title="Direct link to Tying it all together">​</a></h3><p>Now that we have built the stages of our pipeline it&#x27;s time to chain them together into a single model that can be used to process batches of incoming data</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Cog%20Service%20NB/full_pipe_2.jpg" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the final coulmns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cleanupColumns </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;firstCeleb&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentimentLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        celebs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        firstCeleb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        recognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sentimentTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        getSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cleanupColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celebrityQuoteAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Advanced Usage - Async, Batching, and Multi-Key</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#extracting-celebrity-quote-images-using-bing-image-search-on-spark" class="table-of-contents__link toc-highlight">Extracting celebrity quote images using Bing Image Search on Spark</a></li><li><a href="#recognizing-images-of-celebrities" class="table-of-contents__link toc-highlight">Recognizing Images of Celebrities</a></li><li><a href="#reading-the-quote-from-the-image" class="table-of-contents__link toc-highlight">Reading the quote from the image.</a></li><li><a href="#understanding-the-sentiment-of-the-quote" class="table-of-contents__link toc-highlight">Understanding the Sentiment of the Quote</a></li><li><a href="#tying-it-all-together" class="table-of-contents__link toc-highlight">Tying it all together</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html
index fc445a1981..5a052cef58 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Analyze Text/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Analyze Text/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Analyze Text/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Analyze Text</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Analyze Text with SynapseML and Azure AI Language</h1><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/overview" target="_blank" rel="noopener noreferrer">Azure AI Language</a> is a cloud-based service that provides Natural Language Processing (NLP) features for understanding and analyzing text. Use this service to help build intelligent applications using the web-based Language Studio, REST APIs, and client libraries.
 You can use SynapseML with Azure AI Language for <strong>named entity recognition</strong>, <strong>language detection</strong>, <strong>entity linking</strong>, <strong>key phrase extraction</strong>, <strong>Pii entity recognition</strong> and <strong>sentiment analysis</strong>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="named-entity-recognition">Named Entity Recognition<a href="#named-entity-recognition" class="hash-link" aria-label="Direct link to Named Entity Recognition" title="Direct link to Named Entity Recognition">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/overview" target="_blank" rel="noopener noreferrer">Named Entity Recognition</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. The NER feature can identify and categorize entities in unstructured text. For example: people, places, organizations, and quantities. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/named-entity-recognition/language-support?tabs=ga-api" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">language</th><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">en</td><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td align="left">en</td><td align="left">I had a wonderful trip to Seattle last week.</td><td align="left">[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetection">LanguageDetection<a href="#languagedetection" class="hash-link" aria-label="Direct link to LanguageDetection" title="Direct link to LanguageDetection">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/overview" target="_blank" rel="noopener noreferrer">Language detection</a> can detect the language a document is written in. It returns a language code for a wide range of languages, variants, dialects, and some regional/cultural languages. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/language-detection/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a document written in English.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;这是一份用中文写的文件&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language_detection </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LanguageDetection&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> language_detection</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_language.documents.detectedLanguage&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">detectedLanguage</th></tr></thead><tbody><tr><td align="left">This is a document written in English.</td><td align="left">{&quot;name&quot;: &quot;English&quot;, &quot;iso6391Name&quot;: &quot;en&quot;, &quot;confidenceScore&quot;: 0.99}</td></tr><tr><td align="left">这是一份用中文写的文件</td><td align="left">{&quot;name&quot;: &quot;Chinese_Simplified&quot;, &quot;iso6391Name&quot;: &quot;zh_chs&quot;, &quot;confidenceScore&quot;: 1}</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="entitylinking">EntityLinking<a href="#entitylinking" class="hash-link" aria-label="Direct link to EntityLinking" title="Direct link to EntityLinking">​</a></h2><p><a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/overview" target="_blank" rel="noopener noreferrer">Entity linking</a> identifies and disambiguates the identity of entities found in text. For example, in the sentence &quot;We went to Seattle last week.&quot;, the word &quot;Seattle&quot; would be identified, with a link to more information on Wikipedia. <a href="https://learn.microsoft.com/azure/ai-services/language-service/entity-linking/language-support" target="_blank" rel="noopener noreferrer">English and Spanish are supported</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;We went to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_linking </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityLinking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_linking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entity_linking.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[{&quot;bingId&quot;: &quot;a093e9b9-90f5-a3d5-c4b8-5855e1b01f85&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Microsoft&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.48, &quot;length&quot;: 9, &quot;offset&quot;: 0, &quot;text&quot;: &quot;Microsoft&quot;}]<!-- -->, &quot;name&quot;: &quot;Microsoft&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Microsoft%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Microsoft&quot;}</a>, {&quot;bingId&quot;: &quot;0d47c987-0042-5576-15e8-97af601614fa&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Bill Gates&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.52, &quot;length&quot;: 10, &quot;offset&quot;: 25, &quot;text&quot;: &quot;Bill Gates&quot;}]<!-- -->, &quot;name&quot;: &quot;Bill Gates&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Bill_Gates%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Bill_Gates&quot;}</a>, {&quot;bingId&quot;: &quot;df2c4376-9923-6a54-893f-2ee5a5badbc7&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Paul Allen&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.54, &quot;length&quot;: 10, &quot;offset&quot;: 40, &quot;text&quot;: &quot;Paul Allen&quot;}]<!-- -->, &quot;name&quot;: &quot;Paul Allen&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Paul_Allen%22%7D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Paul_Allen&quot;}</a>, {&quot;bingId&quot;: &quot;52535f87-235e-b513-54fe-c03e4233ac6e&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;April 4&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.38, &quot;length&quot;: 7, &quot;offset&quot;: 54, &quot;text&quot;: &quot;April 4&quot;}]<!-- -->, &quot;name&quot;: &quot;April 4&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/April_4%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/April_4&quot;}]</a></td></tr><tr><td align="left">We went to Seattle last week.</td><td align="left">[{&quot;bingId&quot;: &quot;5fbba6b8-85e1-4d41-9444-d9055436e473&quot;, &quot;dataSource&quot;: &quot;Wikipedia&quot;, &quot;id&quot;: &quot;Seattle&quot;, &quot;language&quot;: &quot;en&quot;, &quot;matches&quot;: <!-- -->[{&quot;confidenceScore&quot;: 0.17, &quot;length&quot;: 7, &quot;offset&quot;: 11, &quot;text&quot;: &quot;Seattle&quot;}]<!-- -->, &quot;name&quot;: &quot;Seattle&quot;, &quot;url&quot;: &quot;<a href="https://en.wikipedia.org/wiki/Seattle%22%7D%5D" target="_blank" rel="noopener noreferrer">https://en.wikipedia.org/wiki/Seattle&quot;}]</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextraction">KeyPhraseExtraction<a href="#keyphraseextraction" class="hash-link" aria-label="Direct link to KeyPhraseExtraction" title="Direct link to KeyPhraseExtraction">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/key-phrase-extraction/overview" target="_blank" rel="noopener noreferrer">Key phrase extraction</a> is one of the features offered by Azure AI Language, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. Use key phrase extraction to quickly identify the main concepts in text. For example, in the text &quot;The food was delicious and the staff were wonderful.&quot;, key phrase extraction will return the main topics: &quot;food&quot; and &quot;wonderful staff&quot;. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/key-phrase-extraction/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key_phrase_extraction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;KeyPhraseExtraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> key_phrase_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key_phrase_extraction.documents.keyPhrases&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">keyPhrases</th></tr></thead><tbody><tr><td align="left">Microsoft was founded by Bill Gates and Paul Allen on April 4, 1975.</td><td align="left">[&quot;Bill Gates&quot;, &quot;Paul Allen&quot;, &quot;Microsoft&quot;, &quot;April&quot;]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[&quot;modern medical office&quot;, &quot;Dr. Smith&quot;, &quot;great staff&quot;]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="piientityrecognition">PiiEntityRecognition<a href="#piientityrecognition" class="hash-link" aria-label="Direct link to PiiEntityRecognition" title="Direct link to PiiEntityRecognition">​</a></h2><p>The PII detection feature can identify, categorize, and redact sensitive information in unstructured text. For example: phone numbers, email addresses, and forms of identification. The method for utilizing PII in conversations is different than other use cases, and articles for this use have been separated. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/personally-identifiable-information/language-support?tabs=documents" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Call our office at 312-555-1234, or send an email to support@contoso.com&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii_entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PiiEntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> pii_entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pii_entity_recognition.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">entities</th></tr></thead><tbody><tr><td align="left">Call our office at 312-555-1234, or send an email to <a href="mailto:support@contoso.com" target="_blank" rel="noopener noreferrer">support@contoso.com</a></td><td align="left">[{&quot;category&quot;: &quot;PhoneNumber&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 12, &quot;offset&quot;: 19, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;312-555-1234&quot;}, {&quot;category&quot;: &quot;Email&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 19, &quot;offset&quot;: 53, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;support@contoso.com&quot;}]</td></tr><tr><td align="left">Dr. Smith has a very modern medical office, and she has great staff.</td><td align="left">[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.93, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}]</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sentimentanalysis">SentimentAnalysis<a href="#sentimentanalysis" class="hash-link" aria-label="Direct link to SentimentAnalysis" title="Direct link to SentimentAnalysis">​</a></h2><p><a href="https://learn.microsoft.com/en-us/azure/ai-services/language-service/sentiment-opinion-mining/overview" target="_blank" rel="noopener noreferrer">Sentiment analysis</a> and opinion mining are features offered by the Language service, a collection of machine learning and AI algorithms in the cloud for developing intelligent applications that involve written language. These features help you find out what people think of your brand or topic by mining text for clues about positive or negative sentiment, and can associate them with specific aspects of the text. Refer to <a href="https://learn.microsoft.com/azure/ai-services/language-service/sentiment-opinion-mining/language-support" target="_blank" rel="noopener noreferrer">this article</a> for the full list of supported languages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The food and service were unacceptable. The concierge was nice, however.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;It taste great.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment_analysis </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> sentiment_analysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_analysis.documents.sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">text</th><th align="left">sentiment</th></tr></thead><tbody><tr><td align="left">The food and service were unacceptable. The concierge was nice, however.</td><td align="left">mixed</td></tr><tr><td align="left">It tastes great.</td><td align="left">positive</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="analyze-text-with-textanalyze">Analyze Text with TextAnalyze<a href="#analyze-text-with-textanalyze" class="hash-link" aria-label="Direct link to Analyze Text with TextAnalyze" title="Direct link to Analyze Text with TextAnalyze">​</a></h2><p>Text Analyze is Deprecated, please use AnalyzeText instead</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello Seattle&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;There once was a dog who lived in London and thought she was a human&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">text_analyze </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextAnalyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEntityRecognitionParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model-version&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;latest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Can pass parameters to each model individually</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludePii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Users can manually exclude tasks to speed up analysis</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeEntityLinking</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeSentimentAnalysis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> text_analyze</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Celebrity Quotes</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#named-entity-recognition" class="table-of-contents__link toc-highlight">Named Entity Recognition</a></li><li><a href="#languagedetection" class="table-of-contents__link toc-highlight">LanguageDetection</a></li><li><a href="#entitylinking" class="table-of-contents__link toc-highlight">EntityLinking</a></li><li><a href="#keyphraseextraction" class="table-of-contents__link toc-highlight">KeyPhraseExtraction</a></li><li><a href="#piientityrecognition" class="table-of-contents__link toc-highlight">PiiEntityRecognition</a></li><li><a href="#sentimentanalysis" class="table-of-contents__link toc-highlight">SentimentAnalysis</a></li><li><a href="#analyze-text-with-textanalyze" class="table-of-contents__link toc-highlight">Analyze Text with TextAnalyze</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html
index 618c36014a..bcf5326db9 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create Audiobooks</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Create audiobooks using neural Text to speech</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-load-libraries-and-add-service-information">Step 1: Load libraries and add service information<a href="#step-1-load-libraries-and-add-service-information" class="hash-link" aria-label="Direct link to Step 1: Load libraries and add service information" title="Direct link to Step 1: Load libraries and add service information">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> notebookutils </span><span class="token keyword">import</span><span class="token plain"> mssparkutils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill this in with your Azure AI service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this line with a string like service_key = &quot;dddjnbdkw9329&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_container </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audiobooks&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;madtest-storage-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">storage_account </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalydetectiontest&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-attach-the-storage-account-to-hold-the-audio-files">Step 2: Attach the storage account to hold the audio files<a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="hash-link" aria-label="Direct link to Step 2: Attach the storage account to hold the audio files" title="Direct link to Step 2: Attach the storage account to hold the audio files">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark_key_setting </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;fs.azure.account.key.</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_jsc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoopConfiguration</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path </span><span class="token keyword">import</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> join</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mount_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mount_dir </span><span class="token operator">=</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/synfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">env</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getJobId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> storage_container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mssparkutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accountKey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            source</span><span class="token operator">=</span><span class="token plain">mount_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            mount_point</span><span class="token operator">=</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            extra_configs</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain">spark_key_setting</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> storage_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-in-text-data">Step 3: Read in text data<a href="#step-3-read-in-text-data" class="hash-link" aria-label="Direct link to Step 3: Read in text data" title="Direct link to Step 3: Read in text data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token decorator annotation punctuation" style="color:rgb(249, 38, 114)">@udf</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">part</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;wasbs://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">@</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_account</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.blob.core.windows.net/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">part</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/alice_in_wonderland.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> make_audio_filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;part&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-synthesize-audio-from-text">Step 4: Synthesize audio from text<a href="#step-4-synthesize-audio-from-text" class="hash-link" aria-label="Direct link to Step 4: Synthesize audio from text" title="Direct link to Step 4: Synthesize audio from text">​</a></h2><div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/NeuralTTS_hero.jpeg" width="500" class="img_ev3q"></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">speech </span><span class="token keyword">import</span><span class="token plain"> TextToSpeech</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TextToSpeech</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVoiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US-SteffanNeural&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputFileCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;filename&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audio </span><span class="token operator">=</span><span class="token plain"> tts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-listen-to-an-audio-file">Step 5: Listen to an audio file<a href="#step-5-listen-to-an-audio-file" class="hash-link" aria-label="Direct link to Step 5: Listen to an audio file" title="Direct link to Step 5: Listen to an audio file">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display </span><span class="token keyword">import</span><span class="token plain"> Audio</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/mnt/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">storage_container</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mount_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;alice_in_wonderland/part_</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">num</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Audio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token plain">get_audio_file</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create a Visual Search Engine</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-load-libraries-and-add-service-information" class="table-of-contents__link toc-highlight">Step 1: Load libraries and add service information</a></li><li><a href="#step-2-attach-the-storage-account-to-hold-the-audio-files" class="table-of-contents__link toc-highlight">Step 2: Attach the storage account to hold the audio files</a></li><li><a href="#step-3-read-in-text-data" class="table-of-contents__link toc-highlight">Step 3: Read in text data</a></li><li><a href="#step-4-synthesize-audio-from-text" class="table-of-contents__link toc-highlight">Step 4: Synthesize audio from text</a></li><li><a href="#step-5-listen-to-an-audio-file" class="table-of-contents__link toc-highlight">Step 5: Listen to an audio file</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html
index 3d03f0ff88..0839803315 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Create a Visual Search Engine</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="theme-doc-markdown markdown"><h1>Creating a searchable Art Database with The MET&#x27;s open-access collection</h1><p>In this example, we show how you can enrich data using Cognitive Skills and write to an Azure Search Index using SynapseML. We use a subset of The MET&#x27;s open-access collection and enrich it by passing it through &#x27;Describe Image&#x27; and a custom &#x27;Image Similarity&#x27; skill. The results are then written to a searchable index.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_loc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azure_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/metartworks_sample.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Neighbors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;array&lt;string&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworkSamples.png" width="800" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> SelectColumns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># define pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">describeImage </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PrimaryImageUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    describeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RawImageDescription&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/CognitiveSearchHyperscale/MetArtworksProcessed.png" width="800" class="img_ev3q"><p>Before writing the results to a Search Index, you must define a schema which must specify the name, type, and attributes of each field in your index. Refer <a href="https://docs.microsoft.com/azure/search/search-what-is-an-index" target="_blank" rel="noopener noreferrer">Create a basic index in Azure Search</a> for more information.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ObjectID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The Search Index can be queried using the <a href="https://docs.microsoft.com/rest/api/searchservice/" target="_blank" rel="noopener noreferrer">Azure Search REST API</a> by sending GET or POST requests and specifying query parameters that give the criteria for selecting matching documents. For more information on querying refer <a href="https://docs.microsoft.com/rest/api/searchservice/Search-Documents" target="_blank" rel="noopener noreferrer">Query your Azure Search index using the REST API</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> azure_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Analyze Text</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html
index f8b4e384ec..75ff89a505 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Document Question and Answering with PDFs</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>A Guide to Q&amp;A on PDF Documents</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="introduction">Introduction<a href="#introduction" class="hash-link" aria-label="Direct link to Introduction" title="Direct link to Introduction">​</a></h2><p>In this notebook, we&#x27;ll demonstrate how to develop a context-aware question answering framework for any form of a document using <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">OpenAI models</a>, <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a> and <a href="https://azure.microsoft.com/products/ai-services/" target="_blank" rel="noopener noreferrer">Azure AI Services</a>. In this notebook, we assume that PDF documents are the source of data, however, the same framework can be easiy extended to other document formats too.   </p><p>We’ll cover the following key steps:</p><ol><li>Preprocessing PDF Documents: Learn how to load the PDF documents into a Spark DataFrame, read the documents using the <a href="https://azure.microsoft.com/products/ai-services/ai-document-intelligence" target="_blank" rel="noopener noreferrer">Azure AI Document Intelligence</a> in Azure AI Services, and use SynapseML to split the documents into chunks.</li><li>Embedding Generation and Storage: Learn how to generate embeddings for the chunks using SynapseML and <a href="https://azure.microsoft.com/products/ai-services/openai-service" target="_blank" rel="noopener noreferrer">Azure OpenAI Services</a>, store the embeddings in a vector store using <a href="https://azure.microsoft.com/products/search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a>, and search the vector store to answer the user’s question.</li><li>Question Answering Pipeline: Learn how to retrieve relevant document based on the user’s question and provide the answer using <a href="https://python.langchain.com/en/latest/index.html#" target="_blank" rel="noopener noreferrer">Langchain</a>.</li></ol><p>We start by installing the necessary python libraries.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.<a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="hash-link" aria-label="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications." title="Direct link to Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.">​</a></h3><p>To authenticate Azure AI Services and Azure OpenAI applications, you need to provide the respective API keys. Here is an example of how you can provide the keys in Python code. <code>find_secret()</code> function uses Azure Keyvault to get the API keys, however you can directly paste your own keys there.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_services_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your Azure service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_endpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">aoai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_deployment_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aoai_model_name_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Cognitive Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_index_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examplevectorindex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cogsearch_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-the-pdf-documents-into-a-spark-dataframe">Step 2: Load the PDF documents into a Spark DataFrame.<a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="hash-link" aria-label="Direct link to Step 2: Load the PDF documents into a Spark DataFrame." title="Direct link to Step 2: Load the PDF documents into a Spark DataFrame.">​</a></h3><p>For this tutorial, we will be using NASA&#x27;s <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_book_2019_tagged.pdf" target="_blank" rel="noopener noreferrer">Earth</a> and <a href="https://www.nasa.gov/sites/default/files/atoms/files/earth_at_night_508.pdf" target="_blank" rel="noopener noreferrer">Earth at Night</a> e-books. To load PDF documents into a Spark DataFrame, you can use the <code>spark.read.format(&quot;binaryFile&quot;)</code> method provided by Apache Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">document_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/NASAEarth&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># path to your document</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">document_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code will read the PDF documents and create a Spark DataFrame named df with the contents of the PDFs. The DataFrame will have a schema that represents the structure of the PDF documents, including their textual content.</p><p>Let&#x27;s take a glimpse at the contents of the e-books we are working with. Below are some screenshots that showcase the essence of the books; as you can see they contain information about the Earth.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthbook_screenshot.png" width="500" class="img_ev3q"><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/NASAearthatnight_screenshot.png" width="460" class="img_ev3q"><h5 class="anchor anchorWithStickyNavbar_LWe7" id="display-the-raw-data-from-the-pdf-documents">Display the raw data from the PDF documents<a href="#display-the-raw-data-from-the-pdf-documents" class="hash-link" aria-label="Direct link to Display the raw data from the PDF documents" title="Direct link to Display the raw data from the PDF documents">​</a></h5><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Show the dataframe without the content</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-read-the-documents-using-azure-ai-document-intelligence">Step 3: Read the documents using Azure AI Document Intelligence.<a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="hash-link" aria-label="Direct link to Step 3: Read the documents using Azure AI Document Intelligence." title="Direct link to Step 3: Read the documents using Azure AI Document Intelligence.">​</a></h3><p>We utilize <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">SynapseML</a>, an ecosystem of tools designed to enhance the distributed computing framework <a href="https://github.com/apache/spark" target="_blank" rel="noopener noreferrer">Apache Spark</a>. SynapseML introduces advanced networking capabilities to the Spark ecosystem and offers user-friendly SparkML transformers for various <a href="https://azure.microsoft.com/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a>.</p><p>Additionally, we employ AnalyzeDocument from Azure AI Services to extract the complete document content and present it in the designated columns called &quot;output_content&quot; and &quot;paragraph.&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyze_document </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_services_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageBytesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1-15&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Here we are reading the first 15 pages of the documents for demo purposes</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    analyze_document</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.paragraphs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can observe the analayzed Spark DataFrame named <code>analyzed_df</code> using the following code. Note that we drop the &quot;content&quot; column as it is not needed anymore.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-split-the-documents-into-chunks">Step 4: Split the documents into chunks.<a href="#step-4-split-the-documents-into-chunks" class="hash-link" aria-label="Direct link to Step 4: Split the documents into chunks." title="Direct link to Step 4: Split the documents into chunks.">​</a></h3><p>After analyzing the document, we leverage SynapseML’s PageSplitter to divide the documents into smaller sections, which are subsequently stored in the “chunks” column. This allows for more granular representation and processing of the document content.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> PageSplitter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">splitted_df </span><span class="token operator">=</span><span class="token plain"> ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note that the chunks for each document are presented in a single row inside an array. In order to embed all the chunks in the following cells, we need to have each chunk in a separate row. To accomplish that, we first explode these arrays so there is only one chunk in each row, then filter the Spark DataFrame in order to only keep the path to the document and the chunk in a single row.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Each column contains many chunks for the same document as a vector.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Explode will distribute and replicate the content of a vecor across multple rows</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">exploded_df </span><span class="token operator">=</span><span class="token plain"> splitted_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings.<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings." title="Direct link to Step 5: Generate Embeddings.">​</a></h3><p>To produce embeddings for each chunk, we utilize both SynapseML and Azure OpenAI Service. By integrating the Azure OpenAI service with SynapseML, we can leverage the power of the Apache Spark distributed computing framework to process numerous prompts using the OpenAI service. This integration enables the SynapseML embedding client to generate embeddings in a distributed manner, enabling efficient processing of large volumes of data. If you&#x27;re interested in applying large language models at a distributed scale using Azure OpenAI and Azure Synapse Analytics, you can refer to <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/" target="_blank" rel="noopener noreferrer">this approach</a>. For more detailed information on generating embeddings with Azure OpenAI, you can look <a href="https://learn.microsoft.com/azure/cognitive-services/openai/how-to/embeddings?tabs=console" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">exploded_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-store-the-embeddings-in-azure-cognitive-search-vector-store">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.<a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="hash-link" aria-label="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store." title="Direct link to Step 6: Store the embeddings in Azure Cognitive Search Vector Store.">​</a></h3><p><a href="https://learn.microsoft.com/azure/search/search-what-is-azure-search" target="_blank" rel="noopener noreferrer">Azure Cognitive Search</a> offers a user-friendly interface for creating a vector database, as well as storing and retrieving data using vector search. If you&#x27;re interested in learning more about vector search, you can look <a href="https://github.com/Azure/cognitive-search-vector-pr/tree/main" target="_blank" rel="noopener noreferrer">here</a>.</p><p>Storing data in the AzureCogSearch vector database involves two main steps:</p><p>Creating the Index: The first step is to establish the index or schema of the vector database. This entails defining the structure and properties of the data that will be stored and indexed in the vector database.</p><p>Adding Chunked Documents and Embeddings: The second step involves adding the chunked documents, along with their corresponding embeddings, to the vector datastore. This allows for efficient storage and retrieval of the data using vector search capabilities.</p><p>By following these steps, you can effectively store your chunked documents and their associated embeddings in the AzureCogSearch vector database, enabling seamless retrieval of relevant information through vector search functionality.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># create index ID for ACS</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> writeToAzureSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">cogsearch_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexName</span><span class="token operator">=</span><span class="token plain">cogsearch_index_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idx&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    vectorCols</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dimension&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1536</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-ask-a-question">Step 7: Ask a Question.<a href="#step-7-ask-a-question" class="hash-link" aria-label="Direct link to Step 7: Ask a Question." title="Direct link to Step 7: Ask a Question.">​</a></h3><p>After processing the document, we can proceed to pose a question. We will use <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/" target="_blank" rel="noopener noreferrer">SynapseML</a> to convert the user&#x27;s question into an embedding and then utilize cosine similarity to retrieve the top K document chunks that closely match the user&#x27;s question. It&#x27;s worth mentioning that alternative similarity metrics can also be employed.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_question </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did the astronaut Edgar Mitchell call Earth?&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieve_k </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top 2 documents from vector database</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Ask a question and convert to embeddings</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert question to embedding using synapseML</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aoai_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;questions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errorQ&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_ques_embeddings </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ques</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row </span><span class="token operator">=</span><span class="token plain"> df_ques_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    question_embedding </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> question_embedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Retrieve the top K entries</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.search.windows.net/indexes/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">cogsearch_index_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/docs/search?api-version=2023-07-01-Preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    payload </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dumps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    headers </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Content-Type&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;application/json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> cogsearch_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;POST&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">payload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">status_code</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Generate embeddings for the question and retrieve the top k document chunks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">question_embedding </span><span class="token operator">=</span><span class="token plain"> gen_question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> retrieve_k_chunk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">retrieve_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> question_embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-respond-to-a-users-question">Step 8: Respond to a User’s Question.<a href="#step-8-respond-to-a-users-question" class="hash-link" aria-label="Direct link to Step 8: Respond to a User’s Question." title="Direct link to Step 8: Respond to a User’s Question.">​</a></h3><p>To provide a response to the user&#x27;s question, we will utilize the <a href="https://python.langchain.com/en/latest/index.html" target="_blank" rel="noopener noreferrer">LangChain</a> framework. With the LangChain framework we will augment the retrieved documents with respect to the user&#x27;s question. Following this, we can request a response to the user&#x27;s question from our framework.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import necenssary libraries and setting up OpenAI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> LLMChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> aoai_endpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> aoai_key</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can now wrap up the Q&amp;A journey by asking a question and checking the answer. You will see that Edgar Mitchell called Earth &quot;a sparkling blue and white jewel&quot;!</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define a Question Answering chain function using LangChain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define llm model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        deployment_name</span><span class="token operator">=</span><span class="token plain">aoai_deployment_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model_name</span><span class="token operator">=</span><span class="token plain">aoai_model_name_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_key</span><span class="token operator">=</span><span class="token plain">aoai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        openai_api_version</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Write a preprompt with context and query as variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    context :{context}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer the question based on the context above. If the</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    information to answer the question is not present in the given context then reply &quot;I don&#x27;t know&quot;.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Question: {query}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Answer: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a prompt template</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt_template </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define a chain</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    qa_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> qa_chain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Concatenate the content of retrieved documents</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">context </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chunk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make a Quesion Answer chain function and pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_chain </span><span class="token operator">=</span><span class="token plain"> qa_chain_func</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">answer </span><span class="token operator">=</span><span class="token plain"> qa_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;context&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> context</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> user_question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">answer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Create Audiobooks</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#introduction" class="table-of-contents__link toc-highlight">Introduction</a><ul><li><a href="#step-1-provide-the-keys-for-azure-ai-services-and-azure-openai-to-authenticate-the-applications" class="table-of-contents__link toc-highlight">Step 1: Provide the keys for Azure AI Services and Azure OpenAI to authenticate the applications.</a></li><li><a href="#step-2-load-the-pdf-documents-into-a-spark-dataframe" class="table-of-contents__link toc-highlight">Step 2: Load the PDF documents into a Spark DataFrame.</a></li><li><a href="#step-3-read-the-documents-using-azure-ai-document-intelligence" class="table-of-contents__link toc-highlight">Step 3: Read the documents using Azure AI Document Intelligence.</a></li><li><a href="#step-4-split-the-documents-into-chunks" class="table-of-contents__link toc-highlight">Step 4: Split the documents into chunks.</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings.</a></li><li><a href="#step-6-store-the-embeddings-in-azure-cognitive-search-vector-store" class="table-of-contents__link toc-highlight">Step 6: Store the embeddings in Azure Cognitive Search Vector Store.</a></li><li><a href="#step-7-ask-a-question" class="table-of-contents__link toc-highlight">Step 7: Ask a Question.</a></li><li><a href="#step-8-respond-to-a-users-question" class="table-of-contents__link toc-highlight">Step 8: Respond to a User’s Question.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html
index a1e9576e56..789edc1821 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Flooding Risk</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Visualizing Customer addresses on a flood plane</h1><p>King County (WA) publishes flood plain data as well as tax parcel data. We can use the addresses in the tax parcel data and use the geocoder to calculate coordinates. Using this coordinates and the flood plain data we can enrich out dataset with a flag indicating whether the house is in a flood zone or not.</p><p>The following data has been sourced from King County&#x27;s Open data portal. <a href="https://data.kingcounty.gov/" target="_blank" rel="noopener noreferrer"><em>Link</em></a></p><ol><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyAddress.csv" target="_blank" rel="noopener noreferrer">Address Data</a></li><li><a href="https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson" target="_blank" rel="noopener noreferrer">Flood plains</a></li></ol><p>For this demonstration, please follow the instructions on setting up your azure maps account from the overview notebook.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ol><li>Upload the flood plains data as map data to your creator resource</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapters </span><span class="token keyword">import</span><span class="token plain"> HTTPAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">packages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urllib3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">util</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">retry </span><span class="token keyword">import</span><span class="token plain"> Retry</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Configure more resiliant requests to stop flakiness</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retry_strategy </span><span class="token operator">=</span><span class="token plain"> Retry</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    total</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status_forcelist</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">429</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">502</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">503</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">504</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    allowed_methods</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;HEAD&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DELETE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPTIONS&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRACE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> HTTPAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_retries</span><span class="token operator">=</span><span class="token plain">retry_strategy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Session</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps account key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">maps_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azuremaps-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace this with your azure maps key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Creator Geo prefix</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># for this example, assuming that the creator resource is created in `EAST US 2`.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">atlas_geo_prefix </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;us&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Load flood plains data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_geojson </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/maps/KingCountyFloodPlains.geojson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Upload this flood plains data to your maps/creator account. This is a Long-Running async operation and takes approximately 15~30 seconds to complete</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">r </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/upload?api-version=1.0&amp;dataFormat=geojson&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json</span><span class="token operator">=</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_geojson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Poll for resource upload completion</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">resource_location </span><span class="token operator">=</span><span class="token plain"> r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;location&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resource </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">resource_location</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    status </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lower</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;running&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># wait in a polling loop</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">elif</span><span class="token plain"> status </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;succeeded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">break</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Unknown status {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">status</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Once the above operation returns a HTTP 201, get the user_data_id of the flood plains data, you uploaded to your map account.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id_resource_url </span><span class="token operator">=</span><span class="token plain"> resource</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resourceLocation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">user_data_id </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id_resource_url</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;udid&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have the flood plains data setup in our maps account, we can use the <code>CheckPointInPolygon</code> function to check if a location <code>(lat,lon)</code> coordinate is in a flood zone.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="load-address-data">Load address data:<a href="#load-address-data" class="hash-link" aria-label="Direct link to Load address data:" title="Direct link to Load address data:">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/maps/KingCountyAddress.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Visualize incoming schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Choose a subset of the data for this example</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">subset_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="wire-up-the-address-geocoder">Wire-up the Address Geocoder<a href="#wire-up-the-address-geocoder" class="hash-link" aria-label="Direct link to Wire-up the Address Geocoder" title="Direct link to Wire-up the Address Geocoder">​</a></h3><p>We will use the address geocoder to enrich the dataset with location coordinates of the addresses.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FlattenBatch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">geospatial </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.response.results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;position&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Azure Maps geocoder to enhance the dataframe with location data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AddressGeocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAddressCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FullAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Set up a fixed mini batch transformer to geocode addresses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">batched_dataframe </span><span class="token operator">=</span><span class="token plain"> geocoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">subset_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">geocoded_addresses </span><span class="token operator">=</span><span class="token plain"> extract_location_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batched_dataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have geocoded the addresses, we can now use the <code>CheckPointInPolygon</code> function to check if a property is in a flood zone or not.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup-check-point-in-polygon">Setup Check Point In Polygon<a href="#setup-check-point-in-polygon" class="hash-link" aria-label="Direct link to Setup Check Point In Polygon" title="Direct link to Setup Check Point In Polygon">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use this function to select only lat/lon columns into the dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.pointInPolygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In Polygon&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.result.intersectingGeometries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Intersecting Polygons&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">check_point_in_polygon </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    CheckPointInPolygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maps_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGeography</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">atlas_geo_prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserDataIdentifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">user_data_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLatitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLongitudeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Longitude&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">flood_plain_addresses </span><span class="token operator">=</span><span class="token plain"> extract_point_in_polygon_result_fields</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    check_point_in_polygon</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">geocoded_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Display the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">flood_plain_addresses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanup-uploaded-user-data-optional">Cleanup Uploaded User Data (Optional)<a href="#cleanup-uploaded-user-data-optional" class="hash-link" aria-label="Direct link to Cleanup Uploaded User Data (Optional)" title="Direct link to Cleanup Uploaded User Data (Optional)">​</a></h3><p>You can (optionally) delete the uploaded geojson polygon.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">res </span><span class="token operator">=</span><span class="token plain"> http</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delete</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">atlas_geo_prefix</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.atlas.microsoft.com/mapData/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">user_data_id</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">?api-version=1.0&amp;subscription-key=</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">maps_key</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Document Question and Answering with PDFs</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a><ul><li><a href="#load-address-data" class="table-of-contents__link toc-highlight">Load address data:</a></li><li><a href="#wire-up-the-address-geocoder" class="table-of-contents__link toc-highlight">Wire-up the Address Geocoder</a></li><li><a href="#setup-check-point-in-polygon" class="table-of-contents__link toc-highlight">Setup Check Point In Polygon</a></li><li><a href="#cleanup-uploaded-user-data-optional" class="table-of-contents__link toc-highlight">Cleanup Uploaded User Data (Optional)</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html b/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html
index ab42f167fb..f5663ca9b4 100644
--- a/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
+++ b/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Geospatial Services/">Geospatial Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Multivariate Anomaly Detection/">Multivariate Anomaly Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Advanced Usage - Async, Batching, and Multi-Key/">Advanced Usage - Async, Batching, and Multi-Key</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Celebrity Quotes/">Quickstart - Analyze Celebrity Quotes</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Analyze Text/">Quickstart - Analyze Text</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create a Visual Search Engine/">Quickstart - Create a Visual Search Engine</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Create Audiobooks/">Quickstart - Create Audiobooks</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Document Question and Answering with PDFs/">Quickstart - Document Question and Answering with PDFs</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/">Quickstart - Flooding Risk</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">Quickstart - Predictive Maintenance</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">AI Services</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Predictive Maintenance</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Recipe: Predictive maintenance with the Azure AI Services for Big Data</h1><p>This recipe shows how you can use Azure Synapse Analytics and Azure AI services on Apache Spark for predictive maintenance of IoT devices. We&#x27;ll follow along with the <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">CosmosDB and Synapse Link</a> sample. To keep things simple, in this recipe we&#x27;ll read the data straight from a CSV file rather than getting streamed data through CosmosDB and Synapse Link. We strongly encourage you to look over the Synapse Link sample.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="important">Important<a href="#important" class="hash-link" aria-label="Direct link to Important" title="Direct link to Important">​</a></h2><p>Starting on the 20th of September, 2023 you won’t be able to create new Anomaly Detector resources. The Anomaly Detector service is being retired on the 1st of October, 2026.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="hypothetical-scenario">Hypothetical scenario<a href="#hypothetical-scenario" class="hash-link" aria-label="Direct link to Hypothetical scenario" title="Direct link to Hypothetical scenario">​</a></h2><p>The hypothetical scenario is a Power Plant, where IoT devices are monitoring <a href="https://en.wikipedia.org/wiki/Steam_turbine" target="_blank" rel="noopener noreferrer">steam turbines</a>. The IoTSignals collection has Revolutions per minute (RPM) and Megawatts (MW) data for each turbine. Signals from steam turbines are being analyzed and anomalous signals are detected.</p><p>There could be outliers in the data in random frequency. In those situations, RPM values will go up and MW output will go down, for circuit protection. The idea is to see the data varying at the same time, but with different signals.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>An Azure subscription - <a href="https://azure.microsoft.com/free/" target="_blank" rel="noopener noreferrer">Create one for free</a></li><li><a href="https://docs.microsoft.com/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Azure Synapse workspace</a> configured with a <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark" target="_blank" rel="noopener noreferrer">serverless Apache Spark pool</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-anomaly-detector-resource">Create an Anomaly Detector resource<a href="#create-an-anomaly-detector-resource" class="hash-link" aria-label="Direct link to Create an Anomaly Detector resource" title="Direct link to Create an Anomaly Detector resource">​</a></h3><p>Azure AI Services are represented by Azure resources that you subscribe to. Create a resource for Translator using the <a href="https://docs.microsoft.com/azure/cognitive-services/cognitive-services-apis-create-account?tabs=multiservice%2Clinux" target="_blank" rel="noopener noreferrer">Azure portal</a> or <a href="https://learn.microsoft.com/azure/ai-services/multi-service-resource" target="_blank" rel="noopener noreferrer">Azure CLI</a>. You can also:</p><ul><li>View an existing resource in the  <a href="https://portal.azure.com/" target="_blank" rel="noopener noreferrer">Azure portal</a>.</li></ul><p>Make note of the endpoint and the key for this resource, you&#x27;ll need it in this guide.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="enter-your-service-keys">Enter your service keys<a href="#enter-your-service-keys" class="hash-link" aria-label="Direct link to Enter your service keys" title="Direct link to Enter your service keys">​</a></h2><p>Let&#x27;s start by adding your key and location.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector key here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Paste your anomaly detector location here</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data-into-a-dataframe">Read data into a DataFrame<a href="#read-data-into-a-dataframe" class="hash-link" aria-label="Direct link to Read data into a DataFrame" title="Direct link to Read data into a DataFrame">​</a></h2><p>Next, let&#x27;s read the IoTSignals file into a DataFrame. Open a new notebook in your Synapse workspace and create a DataFrame from the file.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_signals </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iot/IoTSignals.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inferSchema</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-anomaly-detection-using-ai-services-on-spark">Run anomaly detection using AI services on Spark<a href="#run-anomaly-detection-using-ai-services-on-spark" class="hash-link" aria-label="Direct link to Run anomaly detection using AI services on Spark" title="Direct link to Run anomaly detection using AI services on Spark">​</a></h3><p>The goal is to find instances where the signals from the IoT devices were outputting anomalous values so that we can see when something is going wrong and do predictive maintenance. To do that, let&#x27;s use Anomaly Detector on Spark:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly </span><span class="token keyword">import</span><span class="token plain"> SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitivity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;secondly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_signals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unitSymbol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dateTime&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createOrReplaceTempView</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;df_anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s take a look at the data:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;deviceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies.isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This cell should yield a result that looks like:</p><table><thead><tr><th align="left">timestamp</th><th align="right">value</th><th align="left">deviceId</th><th align="left">isAnomaly</th></tr></thead><tbody><tr><td align="left">2020-05-01 18:33:51</td><td align="right">3174</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:52</td><td align="right">2976</td><td align="left">dev-7</td><td align="left">False</td></tr><tr><td align="left">2020-05-01 18:33:53</td><td align="right">2714</td><td align="left">dev-7</td><td align="left">False</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-anomalies-for-one-of-the-devices">Visualize anomalies for one of the devices<a href="#visualize-anomalies-for-one-of-the-devices" class="hash-link" aria-label="Direct link to Visualize anomalies for one of the devices" title="Direct link to Visualize anomalies for one of the devices">​</a></h2><p>IoTSignals.csv has signals from multiple IoT devices. We&#x27;ll focus on a specific device and visualize anomalous outputs from the device.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_anomaly_single_device </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">select</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  timestamp,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  measureValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue + anomalies.upperMargin as expectedUpperValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  anomalies.expectedValue - anomalies.lowerMargin as expectedLowerValue,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  case when anomalies.isAnomaly=true then 1 else 0 end as isAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">from</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  df_anomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">where deviceid = &#x27;dev-1&#x27; and timestamp &lt; &#x27;2020-04-29&#x27;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">order by timestamp</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">limit 200&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now that we have created a dataframe that represents the anomalies for a particular device, we can visualize these anomalies:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adf_subset </span><span class="token operator">=</span><span class="token plain"> df_anomaly_single_device</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isAnomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedUpperValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;UpperMargin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Expected Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;royalblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    adf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;expectedLowerValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Lower Margin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> adf_subset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;measureValue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ro&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Anomaly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RPM Anomalies with Confidence Intervals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>If successful, your output will look like this:</p><p><img loading="lazy" src="https://github.com/MicrosoftDocs/azure-docs/raw/master/articles/cognitive-services/big-data/media/anomaly-output.png" alt="Anomaly Detector Plot" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next-steps">Next steps<a href="#next-steps" class="hash-link" aria-label="Direct link to Next steps" title="Direct link to Next steps">​</a></h2><p>Learn how to do predictive maintenance at scale with Azure AI services, Azure Synapse Analytics, and Azure CosmosDB. For more information, see the full sample on <a href="https://github.com/Azure-Samples/cosmosdb-synapse-link-samples" target="_blank" rel="noopener noreferrer">GitHub</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Flooding Risk/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Flooding Risk</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Langchain</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#important" class="table-of-contents__link toc-highlight">Important</a></li><li><a href="#hypothetical-scenario" class="table-of-contents__link toc-highlight">Hypothetical scenario</a></li><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a><ul><li><a href="#create-an-anomaly-detector-resource" class="table-of-contents__link toc-highlight">Create an Anomaly Detector resource</a></li></ul></li><li><a href="#enter-your-service-keys" class="table-of-contents__link toc-highlight">Enter your service keys</a></li><li><a href="#read-data-into-a-dataframe" class="table-of-contents__link toc-highlight">Read data into a DataFrame</a><ul><li><a href="#run-anomaly-detection-using-ai-services-on-spark" class="table-of-contents__link toc-highlight">Run anomaly detection using AI services on Spark</a></li></ul></li><li><a href="#visualize-anomalies-for-one-of-the-devices" class="table-of-contents__link toc-highlight">Visualize anomalies for one of the devices</a></li><li><a href="#next-steps" class="table-of-contents__link toc-highlight">Next steps</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html b/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html
index 5a29a64874..78f3f730a8 100644
--- a/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
+++ b/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Quickstart - Isolation Forests</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Anomaly Detection</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Isolation Forests</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Multivariate Anomaly Detection with Isolation Forest</h1><p>This article shows how you can use SynapseML on Apache Spark for multivariate anomaly detection. Multivariate anomaly detection allows for the detection of anomalies among many variables or time series, taking into account all the inter-correlations and dependencies between the different variables. In this scenario, we use SynapseML to train an Isolation Forest model for multivariate anomaly detection, and we then use to the trained model to infer multivariate anomalies within a dataset containing synthetic measurements from three IoT sensors.</p><p>To learn more about the Isolation Forest model please refer to the original paper by <a href="https://cs.nju.edu.cn/zhouzh/zhouzh.files/publication/icdm08b.pdf?q=isolation-forest" target="_blank" rel="noopener noreferrer">Liu <em>et al.</em></a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><ul><li>If running on Synapse, you&#x27;ll need to <a href="/SynapseML/docs/next/Use with MLFlow/Overview/">create an AML workspace and set up linked Service</a> and add the following installation cell.</li><li>If running on Fabric, you need to add the following installation cell and attach the notebook to a lakehouse. On the left side of your notebook, select Add to add an existing lakehouse or create a new one.</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %%configure -f</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># {</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#   &quot;name&quot;: &quot;synapseml&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#   &quot;conf&quot;: {</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#       &quot;spark.jars.packages&quot;: &quot;com.microsoft.azure:synapseml_2.12:&lt;THE_SYNAPSEML_VERSION_YOU_WANT&gt;&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#       &quot;spark.jars.repositories&quot;: &quot;https://mmlspark.azureedge.net/maven&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#       &quot;spark.jars.excludes&quot;: &quot;org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#       &quot;spark.yarn.user.classpath.first&quot;: &quot;true&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#       &quot;spark.sql.parquet.enableVectorizedReader&quot;: &quot;false&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#   }</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># }</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install sqlparse raiwidgets interpret</span><span class="token operator">-</span><span class="token plain">community mlflow</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.6</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="library-imports">Library imports<a href="#library-imports" class="hash-link" aria-label="Direct link to Library imports" title="Direct link to Library imports">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># %matplotlib inline</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="input-data">Input data<a href="#input-data" class="hash-link" aria-label="Direct link to Input data" title="Direct link to Input data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Table inputs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># str: the name of the timestamp column in the table</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputCols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># list(str): the names of the input variables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Training Start time, and number of days to use for training:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-02-24T06:00:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainingEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-08T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceStartTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-09T09:30:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to start the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inferenceEndTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-03-20T23:55:00Z&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># datetime: datetime for when to end the training</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Isolation Forest parameters</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">contamination </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.021</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">num_estimators </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_samples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">max_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># MLFlow experiment</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">artifact_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;isolationforest&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation-forest-model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">platform </span><span class="token operator">=</span><span class="token plain"> current_platform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;databricks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/Shared/isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse_internal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment-</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid1</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Fabric</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">platform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;isolation_forest_experiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="read-data">Read data<a href="#read-data" class="hash-link" aria-label="Direct link to Read data" title="Direct link to Read data">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/generated_sample_mvad_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>cast columns to appropriate data types</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">date_format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;yyyy-MM-dd&#x27;T&#x27;HH:mm:ss&#x27;Z&#x27;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_c5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="training-data-preparation">Training data preparation<a href="#training-data-preparation" class="hash-link" aria-label="Direct link to Training data preparation" title="Direct link to Training data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the training window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> trainingStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> trainingEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="test-data-preparation">Test data preparation<a href="#test-data-preparation" class="hash-link" aria-label="Direct link to Test data preparation" title="Direct link to Test data preparation">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># filter to data with timestamps within the inference window</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;=</span><span class="token plain"> inferenceStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">&amp;</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&lt;=</span><span class="token plain"> inferenceEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train-isolation-forest-model">Train Isolation Forest model<a href="#train-isolation-forest-model" class="hash-link" aria-label="Direct link to Train Isolation Forest model" title="Direct link to Train Isolation Forest model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_estimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_samples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">max_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> contamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we create an ML pipeline to train the Isolation Forest model. We also demonstrate how to create an MLFlow experiment and register the trained model.</p><p>Note that MLFlow model registration is strictly only required if accessing the trained model at a later time. For training the model, and performing inferencing in the same notebook, the model object model is sufficient.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tracking_url </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;aml-mlflow-tracking-url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># check link in prerequisites for more information on mlflow tracking url</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tracking_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> artifact_path</span><span class="token operator">=</span><span class="token plain">artifact_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> registered_model_name</span><span class="token operator">=</span><span class="token plain">model_name</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="perform-inferencing">Perform inferencing<a href="#perform-inferencing" class="hash-link" aria-label="Direct link to Perform inferencing" title="Direct link to Perform inferencing">​</a></h2><p>Load the trained Isolation Forest Model</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># if running_on_databricks():</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#     model_version = &lt;your_model_version&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#     model_uri = f&quot;models:/{model_name}/{model_version}&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># elif running_on_synapse_internal():</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#     model_uri = &quot;runs:/{run_id}/{artifact_path}&quot;.format(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         run_id=run.info.run_id, artifact_path=artifact_path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#     )</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = mlflow.spark.load_model(model_uri)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Perform inferencing</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test_pred </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="ml-interpretability">ML interpretability<a href="#ml-interpretability" class="hash-link" aria-label="Direct link to ML interpretability" title="Direct link to ML interpretability">​</a></h2><p>In this section, we use ML interpretability tools to help unpack the contribution of each sensor to the detected anomalies at any point in time.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># we are trying to explain. In this case, we are trying to explain the &quot;outlierScore&quot; output.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Display the dataframe with <code>shapValues</code> column</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test_pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define UDF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec2array </span><span class="token operator">=</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Here, we extract the SHAP values, the original features and the outlier score column. Then we convert it to a Pandas DataFrame for visualization.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset),</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># and each of the following elements represents the SHAP values for each feature</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec2array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> inputCols </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Retrieve local feature importances</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Removing the first element in the list of local importance values (this is the base value or mean output of the background dataset)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Defining a wrapper class with predict method for creating the Explanation Dashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-results">Visualize results<a href="#visualize-results" class="hash-link" aria-label="Direct link to Visualize results" title="Direct link to Visualize results">​</a></h2><p>Visualize anomaly results and feature contribution scores (derived from local feature importance)</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    anoms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">611</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Multivariate Anomaly Detection Results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor1_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">612</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor2_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">613</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ymax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">anoms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymin</span><span class="token operator">=</span><span class="token plain">ymin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ymax</span><span class="token operator">=</span><span class="token plain">ymax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor3_value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">614</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlier score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">615</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tick_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> which</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;both&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> bottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelbottom</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tab:purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bottom</span><span class="token operator">=</span><span class="token plain">rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_1_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sensor_2_localimp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Contribution scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">grid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;y&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">visualize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following plots:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/mvad_results_local_importances.jpg" class="img_ev3q"></p><ul><li>The first 3 plots above show the sensor time series data in the inference window, in orange, green, purple and blue. The red vertical lines show the detected anomalies (<code>prediction</code> = 1). </li><li>The fourth plot shows the outlierScore of all the points, with the <code>minOutlierScore</code> threshold shown by the dotted red horizontal line.</li><li>The last plot shows the contribution scores of each sensor to the <code>outlierScore</code> for that point.</li></ul><p>Plot aggregate feature importance</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;global importance values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When you run the cell above, you will see the following global feature importance plot:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/global-feature-importance.jpg" class="img_ev3q"></p><p>Visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># View the model explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">HyperOpt</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#library-imports" class="table-of-contents__link toc-highlight">Library imports</a></li><li><a href="#input-data" class="table-of-contents__link toc-highlight">Input data</a></li><li><a href="#read-data" class="table-of-contents__link toc-highlight">Read data</a></li><li><a href="#training-data-preparation" class="table-of-contents__link toc-highlight">Training data preparation</a></li><li><a href="#test-data-preparation" class="table-of-contents__link toc-highlight">Test data preparation</a></li><li><a href="#train-isolation-forest-model" class="table-of-contents__link toc-highlight">Train Isolation Forest model</a></li><li><a href="#perform-inferencing" class="table-of-contents__link toc-highlight">Perform inferencing</a></li><li><a href="#ml-interpretability" class="table-of-contents__link toc-highlight">ML interpretability</a></li><li><a href="#visualize-results" class="table-of-contents__link toc-highlight">Visualize results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Causal Inference/Overview/index.html b/docs/next/Explore Algorithms/Causal Inference/Overview/index.html
index 79f0c1ee29..a3ddb2346a 100644
--- a/docs/next/Explore Algorithms/Causal Inference/Overview/index.html	
+++ b/docs/next/Explore Algorithms/Causal Inference/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -33,7 +33,7 @@
 As a result, we look toward causal inference methods that allow us to estimate the treatment effect using observational data.</p><p>The SynapseML causal package implements a technique &quot;Double machine learning&quot;, which can be used to estimate the average treatment effect via machine learning models.
 Unlike regression-based approaches that make strict parametric assumptions, this machine learning-based approach allows us to model non-linear      relationships between the confounders, treatment, and outcome.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h3><p>In PySpark, you can run the <code>DoubleMLEstimator</code> via:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> DoubleMLEstimator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><blockquote><p>Note: all columns except &quot;Treatment&quot; and &quot;Outcome&quot; in your dataset will be used as confounders.</p></blockquote><blockquote><p>Note: For discrete treatment, the treatment column must be <code>int</code> or <code>bool</code>. <code>0</code> and <code>False</code> will be treated as the control group. </p></blockquote><p>After fitting the model, you can get average treatment effect and confidence interval:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For an end to end application, check out the DoubleMLEstimator <a href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">notebook
 example</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#causal-inference-on-apache-spark" class="table-of-contents__link toc-highlight">Causal Inference on Apache Spark</a><ul><li><a href="#what-is-causal-inference" class="table-of-contents__link toc-highlight">What is Causal Inference?</a></li><li><a href="#causal-inference-language" class="table-of-contents__link toc-highlight">Causal Inference language</a></li><li><a href="#causal-inference-and-double-machine-learning" class="table-of-contents__link toc-highlight">Causal Inference and Double machine learning</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html
index 058b220ee1..c6c552effb 100644
--- a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
+++ b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Causal Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>$ spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="get-causal-effects-with-synapseml-doublemlestimator">Get Causal Effects with SynapseML DoubleMLEstimator<a href="#get-causal-effects-with-synapseml-doublemlestimator" class="hash-link" aria-label="Direct link to Get Causal Effects with SynapseML DoubleMLEstimator" title="Direct link to Get Causal Effects with SynapseML DoubleMLEstimator">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get average treatment effect, it returns a numeric value, e.g. 5166.78324</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># It means, on average, customers who received a discount spent $5,166 more on software</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Get treatment effect&#x27;s confidence interval, e.g.  [4765.826181160708, 5371.2817538168965]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li><li><a href="#get-causal-effects-with-synapseml-doublemlestimator" class="table-of-contents__link toc-highlight">Get Causal Effects with SynapseML DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html
index 90cb1a31f5..9053cd5eaf 100644
--- a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
+++ b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Measure Heterogeneous Effects</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Startup Investment Attribution - Understand Outreach Effort&#x27;s Effect&quot;</h1><p><strong>This sample notebook aims to show the application of using SynapseML&#x27;s DoubleMLEstimator for inferring causality using observational data.</strong></p><p>A startup that sells software would like to know whether its outreach efforts were successful in attracting new customers or boosting consumption among existing customers. In other words, they would like to learn the treatment effect of each investment on customers&#x27; software usage.</p><p>In an ideal world, the startup would run several randomized experiments where each customer would receive a random assortment of investments. However, this can be logistically prohibitive or strategically unsound: the startup might not have the resources to design such experiments or they might not want to risk losing out on big opportunities due to lack of incentives.</p><p>In this customer scenario walkthrough, we show how SynapseML causal package can use historical investment data to learn the investment effect.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="background">Background<a href="#background" class="hash-link" aria-label="Direct link to Background" title="Direct link to Background">​</a></h2><p>In this scenario, a startup that sells software provides discounts incentives to its customer. A customer might be given or not.</p><p>The startup has historical data on these investments for 2,000 customers, as well as how much revenue these customers generated in the year after the investments were made. They would like to use this data to learn the optimal incentive policy for each existing or new customer in order to maximize the return on investment (ROI).</p><p>The startup faces a challenge:  the dataset is biased because historically the larger customers received the most incentives. Thus, they need a causal model that can remove the bias.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data">Data<a href="#data" class="hash-link" aria-label="Direct link to Data" title="Direct link to Data">​</a></h2><p>The data* contains ~2,000 customers and is comprised of:</p><ul><li>Customer features: details about the industry, size, revenue, and technology profile of each customer.</li><li>Interventions: information about which incentive was given to a customer.</li><li>Outcome: the amount of product the customer bought in the year after the incentives were given.</li></ul><table><thead><tr><th>Feature Name</th><th>Type</th><th>Details</th></tr></thead><tbody><tr><td>Global Flag</td><td>W</td><td>whether the customer has global offices</td></tr><tr><td>Major Flag</td><td>W</td><td>whether the customer is a large consumer in their industry (as opposed to SMC - Small Medium Corporation - or SMB - Small Medium Business)</td></tr><tr><td>SMC Flag</td><td>W</td><td>whether the customer is a Small Medium Corporation (SMC, as opposed to major and SMB)</td></tr><tr><td>Commercial Flag</td><td>W</td><td>whether the customer&#x27;s business is commercial (as opposed to public secor)</td></tr><tr><td>IT Spend</td><td>W</td><td>dollar spent on IT-related purchases</td></tr><tr><td>Employee Count</td><td>W</td><td>number of employees</td></tr><tr><td>PC Count</td><td>W</td><td>number of PCs used by the customer</td></tr><tr><td>Size</td><td>X</td><td>customer&#x27;s size given by their yearly total revenue</td></tr><tr><td>Discount</td><td>T</td><td>whether the customer was given a discount (binary)</td></tr><tr><td>Revenue</td><td>Y</td><td>$ Revenue from customer given by the amount of software purchased</td></tr></tbody></table><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Import the sample multi-attribution data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/multi_attribution_sample.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Get Heterogenous Causal Effects with SynapseML OrthoDML Estimator</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DateType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">treatmentColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Discount&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outcomeColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Revenue&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Global Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Major Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SMC Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Commercial Flag&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Employee Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PC Count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heteroColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;IT Spend&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVecCol </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;XWVec&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Discount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">heterogeneityVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">heteroColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">heterogeneityVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">confounderVector </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">confounderColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token plain">confounderVecCol</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">heterogeneityVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> confounderVector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ppfit </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Create the Ortho Forest DML Estimator Model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mtTransform </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OrthoForestDMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treatmentColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outcomeColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">heterogeneityVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConfounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">confounderVecCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSamplesLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Fit the model for the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalModel </span><span class="token operator">=</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Transform the input data to see the model in action</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">finalPred </span><span class="token operator">=</span><span class="token plain"> finalModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ppfit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Get the data in Pandas</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd_final </span><span class="token operator">=</span><span class="token plain"> finalPred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">### Plot and see the non-linear effects</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mtTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token operator">=</span><span class="token plain">pd_final</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Causal Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Synthetic difference in differences</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#background" class="table-of-contents__link toc-highlight">Background</a></li><li><a href="#data" class="table-of-contents__link toc-highlight">Data</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html
index c25234e5d8..40377cfc10 100644
--- a/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html	
+++ b/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Causal Effects/">Quickstart - Measure Causal Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/">Quickstart - Measure Heterogeneous Effects</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">Quickstart - Synthetic difference in differences</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Causal Inference</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Synthetic difference in differences</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="theme-doc-markdown markdown"><h1>Scalable Synthetic Difference in Differences</h1><p>This sample notebook aims to show readers how to use SynapseML&#x27;s <code>DiffInDiffEstimator</code>, <code>SyntheticControlEstimator</code> and <code>SyntheticDiffInDiffEstimator</code> to estimate the causal effect of a treatment on a particular outcome.</p><p>In this sample notebook, we will use the California smoking cessation program example to demonstrate usage of the SyntheticDiffInDiff Estimator. The goal of the analysis is to estimate the effect of increased cigarette taxes on smoking in California.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SyntheticControlEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    SyntheticDiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib </span><span class="token keyword">import</span><span class="token plain"> pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib </span><span class="token keyword">import</span><span class="token plain"> style</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLogLevel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;INFO&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">style</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">use</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ggplot&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We will select 5 columns from the dataset: state, year, cigsale, california, after_treatment.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/smoking.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First, we use the <code>DiffInDiffEstimator</code> to estimate the causal effect with regular difference in differences method. We set the treatment indicator column to &quot;california&quot;, set post-treatment indicator column to &quot;after_treatment&quot;, and set the outcome column to &quot;cigsale&quot;.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator1 </span><span class="token operator">=</span><span class="token plain"> DiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model1 </span><span class="token operator">=</span><span class="token plain"> estimator1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Diff in Diff] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Diff in Diff] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by difference in differences should be -27.349.</p><p>Next, we use <code>SyntheticControlEstimator</code> to synthesize a control unit and use the synthetic control to estimate the causal effect. To create the synthetic control unit, we need to set the column which indicates the time when each outcome is measured, and the column which indicates the unit for which the outcome is measured.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator2 </span><span class="token operator">=</span><span class="token plain"> SyntheticControlEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    timeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    unitCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterNoChange</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tol</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1e-4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stepSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model2 </span><span class="token operator">=</span><span class="token plain"> estimator2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Control] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Control] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by synthetic control should be about -19.354.</p><p>Internally, a constrained least square regression is used to solve the unit weights for the synthetic control, and we can plot the loss history.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryUnitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - unit weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also visualize the synthetic control and compare it with the treated unit.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc_weights </span><span class="token operator">=</span><span class="token plain"> model2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdf </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;~california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;California&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control Estimation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cigarette Sales&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1988</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ymin</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ymax</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">140</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Proposition 99&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Lastly, we use <code>SyntheticDiffInDiffEstimator</code> to estimate the causal effect.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator3 </span><span class="token operator">=</span><span class="token plain"> SyntheticDiffInDiffEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    timeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    unitCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    postTreatmentCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;after_treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outcomeCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterNoChange</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tol</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1e-4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stepSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model3 </span><span class="token operator">=</span><span class="token plain"> estimator3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Diff in Diff] treatment effect: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">treatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;[Synthetic Diff in Diff] standard error: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">standardError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The treatment effect estimated by synthetic control should be about -15.554.</p><p>Again, we can plot the loss history from the optimizer used to solve the unit weights and the time weights.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryUnitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - unit weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lossHistory </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lossHistoryTimeWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss history - time weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Iteration&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mimimal loss: {}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lossHistory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Here we plot the synthetic diff in diff estimate together with the time weights.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">unit_weights </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unit_intercept </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unitIntercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time_weights </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">timeWeights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">time_intercept </span><span class="token operator">=</span><span class="token plain"> model3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">timeIntercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdf </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pivot_df_control </span><span class="token operator">=</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;~california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pivot_df_treat </span><span class="token operator">=</span><span class="token plain"> pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;california&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pivot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    index</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;year&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;state&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cigsale&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc_did </span><span class="token operator">=</span><span class="token plain"> pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values @ unit_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">treated_mean </span><span class="token operator">=</span><span class="token plain"> pivot_df_treat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharex</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gridspec_kw</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;height_ratios&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suptitle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Diff in Diff Estimation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dashed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Control Avg.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lw</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;California&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pivot_df_control</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sc_did</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Synthetic Control (SDID)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;C1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cigarette Sales&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1989</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    treated_mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dotted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Prop. 99&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time_weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;skyblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Time Weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Time&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vlines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1989</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ls</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dotted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Measure Heterogeneous Effects/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Measure Heterogeneous Effects</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html b/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html
index a25b8d010c..5da73fffee 100644
--- a/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
+++ b/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -41,7 +41,7 @@
 dataset given the specified metric</p></li><li><p>The <strong><code>ComputeModelStatistics</code></strong> Transformer computes the different
 metrics on a scored dataset (in our case, the <code>validation</code> dataset)
 at the same time</p></li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> FindBestModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Prepare data for learning</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Train the models on the &#x27;train&#x27; data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrHyperParams </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.05</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logisticRegressions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">regParam</span><span class="token operator">=</span><span class="token plain">hyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> hyperParam </span><span class="token keyword">in</span><span class="token plain"> lrHyperParams</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lrmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lrm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> lrm </span><span class="token keyword">in</span><span class="token plain"> logisticRegressions</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">lrmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Get AUC on the validation dataset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">validation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Best model&#x27;s AUC on validation set = &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{0:.2f}%&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">first</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AUC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#read-the-data" class="table-of-contents__link toc-highlight">Read the data</a></li><li><a href="#extract-features-and-process-data" class="table-of-contents__link toc-highlight">Extract features and process data</a></li><li><a href="#classify-using-pyspark" class="table-of-contents__link toc-highlight">Classify using pyspark</a></li><li><a href="#classify-using-synapseml" class="table-of-contents__link toc-highlight">Classify using SynapseML</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html b/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html
index 75d2c6dc06..c056704b34 100644
--- a/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
+++ b/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -25,7 +25,7 @@
 You can use <code>help(synapse.ml.train.TrainClassifier)</code> to view the different parameters.</p><p>Note that it implicitly converts the data into the format expected by the algorithm: tokenize
 and hash strings, one-hot encodes categorical variables, assembles the features into a vector
 and so on.  The parameter <code>numFeatures</code> controls the number of hashed features.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFeatures</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we save the model so it can be used in a scoring program.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">or</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">elif</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">write</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">overwrite</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/AdultCensus.mml&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">current_platform</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> platform not supported&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Quickstart - Synthetic difference in differences/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Synthetic difference in differences</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification---adult-census" class="table-of-contents__link toc-highlight">Classification - Adult Census</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html b/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html
index 92937c364f..093db30d79 100644
--- a/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
+++ b/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Tabular Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="theme-doc-markdown markdown"><h1>Classification - Adult Census using Vowpal Wabbit in SynapseML</h1><p>In this example, we predict incomes from the <em>Adult Census</em> dataset using Vowpal Wabbit (VW) classifier in SynapseML.
 First, we read the data and split it into train and test sets as in this <a href="https://github.com/Microsoft/SynapseML/blob/master/notebooks/Classification%20-%20Adult%20Census.ipynb" target="_blank" rel="noopener noreferrer">example</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, we define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names.
 Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then, we are ready to train the model by fitting the pipeline with the training data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Train the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using <code>ComputeModelStatistics</code> function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - SparkML vs SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html b/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html
index 069b1292a7..46d4a4f836 100644
--- a/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
+++ b/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Quickstart - Train Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - SparkML vs SynapseML/">Quickstart - SparkML vs SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/">Quickstart - Vowpal Wabbit on Tabular Data</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">Quickstart - Vowpal Wabbit on Text Data</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Classification</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Vowpal Wabbit on Text Data</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Twitter Sentiment Classification using Vowpal Wabbit in SynapseML</h1><p>In this example, we show how to build a sentiment classification model using Vowpal Wabbit (VW) in SynapseML. The data set we use to train and evaluate the model is <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data. First, we import a few packages that we need.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> zipfile </span><span class="token keyword">import</span><span class="token plain"> ZipFile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RegexTokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationMetrics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># URL to download the sentiment140 dataset and data file names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_URL </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/twittersentimenttrainingandtestdata.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TRAIN_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;training.1600000.processed.noemoticon.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">TEST_FILENAME </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testdata.manual.2009.06.14.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Folder for storing the downloaded data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">DATA_FOLDER </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Data column names</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">COL_NAMES </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;date&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Text encoding type of the data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ENCODING </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;iso-8859-1&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-preparation">Data Preparation<a href="#data-preparation" class="hash-link" aria-label="Direct link to Data Preparation" title="Direct link to Data Preparation">​</a></h2><p>We use <a href="http://help.sentiment140.com/for-students/?source=post_page---------------------------" target="_blank" rel="noopener noreferrer">Sentiment140</a> twitter data which originated from a Stanford research project to train and evaluate VW classification model on Spark. The same dataset has been used in a previous <a href="https://github.com/Azure-Samples/MachineLearningSamples-TwitterSentimentPrediction" target="_blank" rel="noopener noreferrer">Azure Machine Learning sample</a> on twitter sentiment prediction. Before using the data to build the classification model, we first download and clean up the data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data_folder</span><span class="token operator">=</span><span class="token plain">DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;downloaded_data.zip&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;Download and extract data from url&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> DATA_FOLDER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">makedirs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    downloaded_filepath </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Downloading data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Extracting data...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile </span><span class="token operator">=</span><span class="token plain"> ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">downloaded_filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">close</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Finished data downloading and extraction.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">download_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DATA_URL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Let&#x27;s read the training data into a Spark DataFrame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TRAIN_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can take a look at the training data and check how many samples it has. We should see that there are 1.6 million samples in the training data. There are 6 fields in the training data:</p><ul><li>label: the sentiment of the tweet (0.0 = negative, 2.0 = neutral, 4.0 = positive)</li><li>id: the id of the tweet</li><li>date: the date of the tweet</li><li>query_string: The query used to extract the data. If there is no query, then this value is NO_QUERY.</li><li>user: the user that tweeted</li><li>text: the text of the tweet</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of training samples: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Before training the model, we randomly permute the data to mix negative and positive samples. This is helpful for properly training online learning algorithms like VW. To speed up model training, we use a subset of the data to train the model. If training with the full training set, typically you will see better performance of the model on the test set. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-synapseml-training">VW SynapseML Training<a href="#vw-synapseml-training" class="hash-link" aria-label="Direct link to VW SynapseML Training" title="Direct link to VW SynapseML Training">​</a></h2><p>Now we are ready to define a pipeline which consists of feature engineering steps and the VW model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tokenizer </span><span class="token operator">=</span><span class="token plain"> RegexTokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">count_vectorizer </span><span class="token operator">=</span><span class="token plain"> CountVectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> count_vectorizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With the prepared training data, we can fit the model pipeline as follows.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="model-performance-evaluation">Model Performance Evaluation<a href="#model-performance-evaluation" class="hash-link" aria-label="Direct link to Model Performance Evaluation" title="Direct link to Model Performance Evaluation">​</a></h2><p>After training the model, we evaluate the performance of the model using the test set which is manually labeled.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DATA_FOLDER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TEST_FILENAME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names</span><span class="token operator">=</span><span class="token plain">COL_NAMES</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    encoding</span><span class="token operator">=</span><span class="token plain">ENCODING</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verifySchema</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We only use positive and negative tweets in the test set to evaluate the model, since our model is a binary classification model trained with only positive and negative tweets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples before filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">!=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of test samples after filtering: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Make predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Compute model performance metrics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Utility class for plotting ROC curve (https://stackoverflow.com/questions/52847408/pyspark-extract-roc-curve)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BinaryClassificationMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token builtin" style="color:rgb(174, 129, 255)">super</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        rdd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">getattr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_java_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> method</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toJavaRDD</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        points </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            points </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> points</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rdd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">roc_points </span><span class="token operator">=</span><span class="token plain"> CurveMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_curve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;roc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Plot ROC curve</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y_val </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> roc_points</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ROC curve on test set&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;False positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True positive rate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Use display() if you&#x27;re on Azure Databricks or you can do plt.show()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You should see an ROC curve like the following after the above cell is executed. </p><img loading="lazy" src="https://user-images.githubusercontent.com/20047467/69376052-9b0a3380-0c77-11ea-9266-11aa44350cbe.png" width="400" height="320" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Tabular Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Tabular Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-preparation" class="table-of-contents__link toc-highlight">Data Preparation</a></li><li><a href="#vw-synapseml-training" class="table-of-contents__link toc-highlight">VW SynapseML Training</a></li><li><a href="#model-performance-evaluation" class="table-of-contents__link toc-highlight">Model Performance Evaluation</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Distributed Training/index.html b/docs/next/Explore Algorithms/Deep Learning/Distributed Training/index.html
index 08a9239087..ecd19bc419 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Distributed Training/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Distributed Training/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -35,7 +35,7 @@
 make it an excellent choice for SynapseML&#x27;s Simple Deep Learning:</p><ul><li>Code Organization: PyTorch Lightning promotes a clean and organized code structure by separating the research code from the engineering code. This property makes it easier to maintain, debug, and share deep learning models.</li><li>Flexibility: PyTorch Lightning retains the flexibility and expressiveness of PyTorch while adding useful abstractions to simplify the training loop and other boilerplate code.</li><li>Built-in Best Practices: PyTorch Lightning incorporates many best practices for deep learning, such as automatic optimization, gradient clipping, and learning rate scheduling, making it easier for users to achieve optimal performance.</li><li>Compatibility: PyTorch Lightning is compatible with a wide range of popular tools and frameworks, including Horovod, which allows users to easily use distributed training capabilities.</li><li>Rapid Development: With PyTorch Lightning, users can quickly experiment with different model architectures and training strategies without worrying about low-level implementation details.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sample-usage-with-deepvisionclassifier">Sample usage with DeepVisionClassifier<a href="#sample-usage-with-deepvisionclassifier" class="hash-link" aria-label="Direct link to Sample usage with DeepVisionClassifier" title="Direct link to Sample usage with DeepVisionClassifier">​</a></h3><p>DeepVisionClassifier incorporates all models supported by <a href="https://github.com/pytorch/vision" target="_blank" rel="noopener noreferrer">torchvision</a>. </p><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>The current version is based on pytorch_lightning v1.5.0 and torchvision v0.12.0</p></div></div><p>By providing a spark dataframe that contains an &#x27;imageCol&#x27; and &#x27;labelCol&#x27;, you could directly apply &#x27;transform&#x27; function
 on it with DeepVisionClassifier.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataframe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PATH_TO_IMAGE_2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Put your backbone here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Corresponding store</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Optional callbacks</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>DeepVisionClassifier does distributed-training on spark with Horovod under the hood, after this fitting process it returns
 a DeepVisionModel. With this code you could use the model for inference directly:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">ONNX</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#why-simple-deep-learning" class="table-of-contents__link toc-highlight">Why Simple Deep Learning</a></li><li><a href="#synapsemls-simple-dnn" class="table-of-contents__link toc-highlight">SynapseML&#39;s Simple DNN</a></li><li><a href="#why-horovod" class="table-of-contents__link toc-highlight">Why Horovod</a></li><li><a href="#why-pytorch-lightning" class="table-of-contents__link toc-highlight">Why Pytorch Lightning</a></li><li><a href="#sample-usage-with-deepvisionclassifier" class="table-of-contents__link toc-highlight">Sample usage with DeepVisionClassifier</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Getting Started/index.html b/docs/next/Explore Algorithms/Deep Learning/Getting Started/index.html
index 3f564e1733..b5c137be82 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Getting Started/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Getting Started/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 Run the following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pip install synapseml==1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>An alternative is installing the SynapseML jar package in library management section, by adding:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Coordinate: com.microsoft.azure:synapseml_2.12:1.0.4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">Repository: https://mmlspark.azureedge.net/maven</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>If you install the jar package, follow the first two cells of this <a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/#environment-setup----reinstall-horovod-based-on-new-version-of-pytorch">sample</a>
 to ensure horovod recognizes SynapseML.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3-try-our-sample-notebook">3. Try our sample notebook<a href="#3-try-our-sample-notebook" class="hash-link" aria-label="Direct link to 3. Try our sample notebook" title="Direct link to 3. Try our sample notebook">​</a></h2><p>You could follow the rest of this <!-- -->[sample]<!-- -->(../Quickstart%20-%20Fine-Tune a Vision Classifier) and have a try on your own dataset.</p><p>Supported models (<code>backbone</code> parameter for <code>DeepVisionClassifer</code>) should be string format of <a href="https://github.com/pytorch/vision/blob/v0.12.0/torchvision/models/__init__.py" target="_blank" rel="noopener noreferrer">Torchvision-supported models</a>;
 You could also check by running <code>backbone in torchvision.models.__dict__</code>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">ONNX</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1-reinstall-horovod-using-our-prepared-script" class="table-of-contents__link toc-highlight">1. Reinstall horovod using our prepared script</a></li><li><a href="#2-install-synapseml-deep-learning-component" class="table-of-contents__link toc-highlight">2. Install SynapseML Deep Learning Component</a></li><li><a href="#3-try-our-sample-notebook" class="table-of-contents__link toc-highlight">3. Try our sample notebook</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/ONNX/index.html b/docs/next/Explore Algorithms/Deep Learning/ONNX/index.html
index 7d719cf54a..65d082ca64 100644
--- a/docs/next/Explore Algorithms/Deep Learning/ONNX/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/ONNX/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 keeping only parts of the model that are needed for those nodes. This new model&#x27;s outputs will be the outputs from
 the intermediate nodes. You can save the sliced model and use it to transform just like any other ONNXModel.</p><p>This slicing feature is used implicitly by the ImageFeaturizer, which uses ONNX models. The OnnxHub manifest entry for each model
 includes which intermediate node outputs should be used for featurization, so the ImageFeaturizer will automatically slice at the correct nodes.</p><p>The below example shows how to perform the slicing manually with a direct ONNXModel.</p><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">    // create a df: Dataframe with image data</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val hub = new ONNXHub()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val info = hub.getModelInfo(&quot;resnet50&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val bytes = hub.load(name)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intermediateOutputName = &quot;resnetv24_pool1_fwd&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModel = new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModelPayload(bytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeedDict(Map(&quot;data&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFetchDict(Map(&quot;rawFeatures&quot; -&gt; intermediateOutputName)) // automatic slicing based on fetch dictionary</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      //   -- or --</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      // .sliceAtOutput(intermediateOutputName) // manual slicing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val slicedModelDf = slicedModel.transform(df)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example">Example<a href="#example" class="hash-link" aria-label="Direct link to Example" title="Direct link to Example">​</a></h2><ul><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Getting Started</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Distributed Training</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnx" class="table-of-contents__link toc-highlight">ONNX</a></li><li><a href="#onnxhub" class="table-of-contents__link toc-highlight">ONNXHub</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#model-slicing" class="table-of-contents__link toc-highlight"><a name="slicing"></a>Model Slicing</a></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html
index eec4b432f6..af94d93e07 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Text Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-text-classifier">Deep Learning - Deep Text Classifier<a href="#deep-learning---deep-text-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Text Classifier" title="Direct link to Deep Learning - Deep Text Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/text_classification/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read_csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/Emotion_classification.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer_model </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> indexer_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">checkpoint </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bert-base-uncased&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">checkpoint</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_classifier </span><span class="token operator">=</span><span class="token plain"> DeepTextClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    checkpoint</span><span class="token operator">=</span><span class="token plain">checkpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_text_model </span><span class="token operator">=</span><span class="token plain"> deep_text_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_text_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Distributed Training</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-text-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Text Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html
index ccdea2db33..91ec2b1bf9 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Fine-tune a Vision Classifier</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---deep-vision-classifier">Deep Learning - Deep Vision Classifier<a href="#deep-learning---deep-vision-classifier" class="hash-link" aria-label="Direct link to Deep Learning - Deep Vision Classifier" title="Direct link to Deep Learning - Deep Vision Classifier">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="environment-setup-on-databricks">Environment Setup on databricks<a href="#environment-setup-on-databricks" class="hash-link" aria-label="Direct link to Environment Setup on databricks" title="Direct link to Environment Setup on databricks">​</a></h3><h3 class="anchor anchorWithStickyNavbar_LWe7" id="---reinstall-horovod-based-on-new-version-of-pytorch">-- reinstall horovod based on new version of pytorch<a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="hash-link" aria-label="Direct link to -- reinstall horovod based on new version of pytorch" title="Direct link to -- reinstall horovod based on new version of pytorch">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># install cloudpickle 2.0.0 to add synapse module for usage of horovod</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token operator">%</span><span class="token plain">pip install cloudpickle</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">force</span><span class="token operator">-</span><span class="token plain">reinstall </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">no</span><span class="token operator">-</span><span class="token plain">deps</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> cloudpickle</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> os</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> zipfile</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cloudpickle</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">register_pickle_by_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">! horovodrun </span><span class="token operator">-</span><span class="token operator">-</span><span class="token plain">check</span><span class="token operator">-</span><span class="token plain">build</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> MulticlassClassificationEvaluator</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read Dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read Dataset" title="Direct link to Read Dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">folder_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/flowers_prepped&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">zip_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Flowers/flowers_prepped.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">zip_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs/tmp/flowers_prepped.zip&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> folder_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlretrieve</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">zip_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zip_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> zipfile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZipFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">zip_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> zip_ref</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        zip_ref</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">extractall</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs/tmp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">remove</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">zip_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> num </span><span class="token operator">//</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">81</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">assign_label_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">assign_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># These files are already uploaded for build test machine</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">folder_path </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/train&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pathGlobFilter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">folder_path </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regexp_replace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/dbfs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> assign_label_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="training">Training<a href="#training" class="hash-link" aria-label="Direct link to Training" title="Direct link to Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> horovod</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">store </span><span class="token keyword">import</span><span class="token plain"> DBFSLocalStore</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pytorch_lightning</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">callbacks </span><span class="token keyword">import</span><span class="token plain"> ModelCheckpoint</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">run_output_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;/dbfs/FileStore/test/resnet50/</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation builtin" style="color:rgb(174, 129, 255)">str</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation">uuid</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">uuid4</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token string-interpolation interpolation format-spec">8]</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">store </span><span class="token operator">=</span><span class="token plain"> DBFSLocalStore</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">epochs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">callbacks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">ModelCheckpoint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{epoch}-{train_loss:.2f}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_classifier </span><span class="token operator">=</span><span class="token plain"> DeepVisionClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backbone</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    store</span><span class="token operator">=</span><span class="token plain">store</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    callbacks</span><span class="token operator">=</span><span class="token plain">callbacks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_classes</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">17</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    batch_size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    epochs</span><span class="token operator">=</span><span class="token plain">epochs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    validation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deep_vision_model </span><span class="token operator">=</span><span class="token plain"> deep_vision_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prediction">Prediction<a href="#prediction" class="hash-link" aria-label="Direct link to Prediction" title="Direct link to Prediction">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pred_df </span><span class="token operator">=</span><span class="token plain"> deep_vision_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> MulticlassClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metricName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Test accuracy:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Cleanup the output dir for test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dbutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">run_output_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Text Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---deep-vision-classifier" class="table-of-contents__link toc-highlight">Deep Learning - Deep Vision Classifier</a><ul><li><a href="#environment-setup-on-databricks" class="table-of-contents__link toc-highlight">Environment Setup on databricks</a></li><li><a href="#---reinstall-horovod-based-on-new-version-of-pytorch" class="table-of-contents__link toc-highlight">-- reinstall horovod based on new version of pytorch</a></li><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read Dataset</a></li><li><a href="#training" class="table-of-contents__link toc-highlight">Training</a></li><li><a href="#prediction" class="table-of-contents__link toc-highlight">Prediction</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html b/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html
index 4c57ea6902..16aa652d4c 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - ONNX Model Inference</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>ONNX Inference on Spark</h1><p>In this example, you train a LightGBM model and convert the model to <a href="https://onnx.ai/" target="_blank" rel="noopener noreferrer">ONNX</a> format. Once converted, you use the model to infer some testing data on Spark.</p><p>This example uses the following Python packages and versions:</p><ul><li><code>onnxmltools==1.7.0</code></li><li><code>lightgbm==3.2.1</code></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-the-example-data">Load the example data<a href="#load-the-example-data" class="hash-link" aria-label="Direct link to Load the example data" title="Direct link to Load the example data">​</a></h2><p>To load the example data, add the following code examples to cells in your notebook and then run the cells:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install lightgbm onnxmltools</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.7</span><span class="token number" style="color:rgb(174, 129, 255)">.0</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Interest Coverage Ratio</th><th>Net Income Flag</th><th>Equity to Liability</th></tr></thead><tbody><tr><td>0.5641</td><td>1.0</td><td>0.0165</td></tr><tr><td>0.5702</td><td>1.0</td><td>0.0208</td></tr><tr><td>0.5673</td><td>1.0</td><td>0.0165</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbm-to-train-a-model">Use LightGBM to train a model<a href="#use-lightgbm-to-train-a-model" class="hash-link" aria-label="Direct link to Use LightGBM to train a model" title="Direct link to Use LightGBM to train a model">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEarlyStoppingRound</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLambdaL1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumThreads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDeltaStep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeatureFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaggingFreq</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIsUnbalance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinSumHessianInLeaf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinGainToSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="convert-the-model-to-onnx-format">Convert the model to ONNX format<a href="#convert-the-model-to-onnx-format" class="hash-link" aria-label="Direct link to Convert the model to ONNX format" title="Direct link to Convert the model to ONNX format">​</a></h2><p>The following code exports the trained model to a LightGBM booster and then converts it to ONNX format:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_binder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_binder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> IPython </span><span class="token keyword">import</span><span class="token plain"> get_ipython</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> lightgbm </span><span class="token keyword">as</span><span class="token plain"> lgb</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> lightgbm </span><span class="token keyword">import</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> LGBMClassifier </span><span class="token keyword">or</span><span class="token plain"> Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxmltools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert </span><span class="token keyword">import</span><span class="token plain"> convert_lightgbm</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> onnxconverter_common</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data_types </span><span class="token keyword">import</span><span class="token plain"> FloatTensorType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    initial_types </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatTensorType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> input_size</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_model </span><span class="token operator">=</span><span class="token plain"> convert_lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initial_types</span><span class="token operator">=</span><span class="token plain">initial_types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target_opset</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> onnx_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SerializeToString</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster_model_str </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getLightGBMBooster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">modelStr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">booster </span><span class="token operator">=</span><span class="token plain"> lgb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_str</span><span class="token operator">=</span><span class="token plain">booster_model_str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_payload_ml </span><span class="token operator">=</span><span class="token plain"> convertModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">booster</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After conversion, load the ONNX payload into an <code>ONNXModel</code> and inspect the model inputs and outputs:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_payload_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model inputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelInputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Model outputs:&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getModelOutputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Map the model input to the input dataframe&#x27;s column name (FeedDict), and map the output dataframe&#x27;s column names to the model outputs (FetchDict).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeviceType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CPU&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probabilities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-the-model-for-inference">Use the model for inference<a href="#use-the-model-for-inference" class="hash-link" aria-label="Direct link to Use the model for inference" title="Direct link to Use the model for inference">​</a></h2><p>To perform inference with the model, the following code creates test data and transforms the data through the ONNX model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">n </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">m </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> m</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testPdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testPdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">onnx_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>The output should look similar to the following table, though the values and number of rows may differ:</p><table><thead><tr><th>Index</th><th>Features</th><th>Prediction</th><th>Probability</th></tr></thead><tbody><tr><td>1</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.105...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.835...</code></td></tr><tr><td>2</td><td><code>&quot;{&quot;type&quot;:1,&quot;values&quot;:[0.814...</code></td><td>0</td><td><code>&quot;{&quot;0&quot;:0.658...</code></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Fine-tune a Vision Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-the-example-data" class="table-of-contents__link toc-highlight">Load the example data</a></li><li><a href="#use-lightgbm-to-train-a-model" class="table-of-contents__link toc-highlight">Use LightGBM to train a model</a></li><li><a href="#convert-the-model-to-onnx-format" class="table-of-contents__link toc-highlight">Convert the model to ONNX format</a></li><li><a href="#use-the-model-for-inference" class="table-of-contents__link toc-highlight">Use the model for inference</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html
index 4c04086ded..16498f89ee 100644
--- a/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
+++ b/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Getting Started</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/ONNX/">ONNX</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Distributed Training/">Distributed Training</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Text Classifier/">Quickstart - Fine-tune a Text Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Fine-tune a Vision Classifier/">Quickstart - Fine-tune a Vision Classifier</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/">Quickstart - ONNX Model Inference</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">Quickstart - Transfer Learn for Image Classification</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Deep Learning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Transfer Learn for Image Classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="deep-learning---flower-image-classification">Deep Learning - Flower Image Classification<a href="#deep-learning---flower-image-classification" class="hash-link" aria-label="Direct link to Deep Learning - Flower Image Classification" title="Direct link to Deep Learning - Flower Image Classification">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Load the images</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># use flowers_and_labels.parquet on larger cluster in order to get better results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/flowers_and_labels2.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bytes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/p2KgdYL.jpg" alt="Smiley face" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Make some featurizers</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ur </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc1 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;scaled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr1 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc2 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">it</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ur</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">resnet </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc3 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc4 </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepModel </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">resnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dc4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://i.imgur.com/Mb4Dyou.png" alt="Resnet 18" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="how-does-it-work">How does it work?<a href="#how-does-it-work" class="hash-link" aria-label="Direct link to How does it work?" title="Direct link to How does it work?">​</a></h3><p><img loading="lazy" src="http://i.stack.imgur.com/Hl2H6.png" alt="Convolutional network weights" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="run-the-experiment">Run the experiment<a href="#run-the-experiment" class="hash-link" aria-label="Direct link to Run the experiment" title="Direct link to Run the experiment">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    start </span><span class="token operator">=</span><span class="token plain"> time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Experiment took {}s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> start</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> result</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> imagesWithLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">basicResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">deepResults </span><span class="token operator">=</span><span class="token plain"> timedExperiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="plot-confusion-matrix">Plot confusion matrix.<a href="#plot-confusion-matrix" class="hash-link" aria-label="Direct link to Plot confusion matrix." title="Direct link to Plot confusion matrix.">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">metrics </span><span class="token keyword">import</span><span class="token plain"> confusion_matrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat </span><span class="token operator">=</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">l</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> l </span><span class="token keyword">in</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    accuracy </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> pred </span><span class="token operator">==</span><span class="token plain"> true </span><span class="token keyword">else</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> confusion_matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y_hat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cm </span><span class="token operator">=</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">sum</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">newaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Accuracy$ $=$ ${}\%$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">accuracy </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> interpolation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;nearest&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Blues</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Predicted$ $label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$True$ $Label$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;$Normalized$ $CM$ $for$ ${}$&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deepResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CNTKModel + LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">basicResults</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LR&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Note that on the larger dataset the accuracy will bump up from 44% to &gt;90%</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - ONNX Model Inference/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - ONNX Model Inference</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#deep-learning---flower-image-classification" class="table-of-contents__link toc-highlight">Deep Learning - Flower Image Classification</a><ul><li><a href="#how-does-it-work" class="table-of-contents__link toc-highlight">How does it work?</a></li><li><a href="#run-the-experiment" class="table-of-contents__link toc-highlight">Run the experiment</a></li><li><a href="#plot-confusion-matrix" class="table-of-contents__link toc-highlight">Plot confusion matrix.</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html b/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html
index c2ec3846f6..1924bb9563 100644
--- a/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
+++ b/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -26,7 +26,7 @@
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> time</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">except</span><span class="token plain"> EOFError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Encountered EOFError while downloading, retrying once...&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    time</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sleep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the data set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-function-to-train-a-model">Create a function to train a model<a href="#create-a-function-to-train-a-model" class="hash-link" aria-label="Direct link to Create a function to train a model" title="Direct link to Create a function to train a model">​</a></h3><p>In this section, you define a function to train a gradient boosting model with SynapseML LightgbmRegressor.  Wrapping the training code in a function is important for passing the function to Hyperopt for tuning later.</p><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert features into a single vector column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> validation_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Using one partition since the training dataset is very small</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This train() function:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - takes hyperparameters as inputs (for tuning later)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">     - returns the F1 score on the validation dataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    Wrapping code as a function makes it easier to reuse the code later with Hyperopt.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use MLflow to track training.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify &quot;nested=True&quot; since this single model will be logged as a child run of Hyperopt&#x27;s run.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nested</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            alpha</span><span class="token operator">=</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            learningRate</span><span class="token operator">=</span><span class="token plain">learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numLeaves</span><span class="token operator">=</span><span class="token plain">numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numIterations</span><span class="token operator">=</span><span class="token plain">numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        metrics </span><span class="token operator">=</span><span class="token plain"> cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># log metrics with mlflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RMSE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;root_mean_squared_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MAE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_absolute_error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run the training function to make sure it works.
 It&#x27;s a good idea to make sure training code runs before adding in tuning.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_metric </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;The trained decision tree achieved a R^2 of </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">val_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> on the validation data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="part-2-use-hyperopt-to-tune-hyperparameters">Part 2. Use Hyperopt to tune hyperparameters<a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="hash-link" aria-label="Direct link to Part 2. Use Hyperopt to tune hyperparameters" title="Direct link to Part 2. Use Hyperopt to tune hyperparameters">​</a></h2><p>In the second section, the Hyperopt workflow is created by:</p><ul><li>Define a function to minimize</li><li>Define a search space over hyperparameters</li><li>Specifying the search algorithm and using <code>fmin()</code> for tuning the model.</li></ul><p>For more information about the Hyperopt APIs, see the <a href="http://hyperopt.github.io/hyperopt/" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-a-function-to-minimize">Define a function to minimize<a href="#define-a-function-to-minimize" class="hash-link" aria-label="Direct link to Define a function to minimize" title="Direct link to Define a function to minimize">​</a></h3><ul><li>Input: hyperparameters</li><li>Internally: Reuse the training function defined above.</li><li>Output: loss</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> hyperopt </span><span class="token keyword">import</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Trials</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> STATUS_OK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    An example train method that calls into MLlib.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    This method is passed to hyperopt.fmin().</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :param params: hyperparameters as a dict. Its structure is consistent with how search space is defined. See below.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    :return: dict with fields &#x27;loss&#x27; (scalar loss) and &#x27;status&#x27; (success/failure status of run)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># For integer parameters, make sure to convert them to int type if Hyperopt is searching over a continuous range of values.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate </span><span class="token operator">=</span><span class="token plain"> params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Hyperopt expects you to return a loss (for which lower is better), so take the negative of the R^2 (for which higher is better).</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loss </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain">r_squared</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> loss</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> STATUS_OK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="define-the-search-space-over-hyperparameters">Define the search space over hyperparameters<a href="#define-the-search-space-over-hyperparameters" class="hash-link" aria-label="Direct link to Define the search space over hyperparameters" title="Direct link to Define the search space over hyperparameters">​</a></h3><p>This example tunes four hyperparameters: <code>alpha</code>, <code>learningRate</code>, <code>numLeaves</code> and <code>numIterations</code>. See the <a href="https://github.com/hyperopt/hyperopt/wiki/FMin#21-parameter-expressions" target="_blank" rel="noopener noreferrer">Hyperopt documentation</a> for details on defining a search space and parameter expressions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">space </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> hp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">uniformint</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tune-the-model-using-hyperopt-fmin">Tune the model using Hyperopt <code>fmin()</code><a href="#tune-the-model-using-hyperopt-fmin" class="hash-link" aria-label="Direct link to tune-the-model-using-hyperopt-fmin" title="Direct link to tune-the-model-using-hyperopt-fmin">​</a></h3><p>For tuning the model with Hyperopt&#x27;s <code>fmin()</code>, the following steps are taken:</p><ul><li>Setting <code>max_evals</code> to the maximum number of points in the hyperparameter space to be tested.</li><li>Specifying the search algorithm, either <code>hyperopt.tpe.suggest</code> or <code>hyperopt.rand.suggest</code>.<ul><li><code>hyperopt.tpe.suggest</code>: Tree of Parzen Estimators, a Bayesian approach which iteratively and adaptively selects new hyperparameter settings to explore based on previous results</li><li><code>hyperopt.rand.suggest</code>: Random search, a non-adaptive approach that randomly samples the search space</li></ul></li></ul><p><strong>Important:</strong><br>
 <!-- -->When using Hyperopt with SynapseML and other distributed training algorithms, do not pass a <code>trials</code> argument to <code>fmin()</code>. When you do not include the <code>trials</code> argument, Hyperopt uses the default <code>Trials</code> class, which runs on the cluster driver. Hyperopt needs to evaluate each trial on the driver node so that each trial can initiate distributed training jobs.  </p><p>Do not use the <code>SparkTrials</code> class with SynapseML. <code>SparkTrials</code> is designed to distribute trials for algorithms that are not themselves distributed. SynapseML uses distributed computing already and is not compatible with <code>SparkTrials</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> tpe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">suggest</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_params </span><span class="token operator">=</span><span class="token plain"> fmin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fn</span><span class="token operator">=</span><span class="token plain">train_with_hyperopt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> space</span><span class="token operator">=</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> algo</span><span class="token operator">=</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_evals</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Print out the parameters that produced the best model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_params</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="retrain-the-model-on-the-full-training-dataset">Retrain the model on the full training dataset<a href="#retrain-the-model-on-the-full-training-dataset" class="hash-link" aria-label="Direct link to Retrain the model on the full training dataset" title="Direct link to Retrain the model on the full training dataset">​</a></h3><p>For tuning, this workflow split the training dataset into training and validation subsets. Now, retrain the model using the &quot;best&quot; hyperparameters on the full training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_alpha </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alpha&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_learningRate </span><span class="token operator">=</span><span class="token plain"> best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;learningRate&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numIterations </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numIterations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">best_numLeaves </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">best_params</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numLeaves&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> val_r_squared </span><span class="token operator">=</span><span class="token plain"> train_tree</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    best_alpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_learningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> best_numLeaves</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the test dataset to compare evaluation metrics for the initial and &quot;best&quot; models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define an evaluation metric and evaluate the model on the test dataset.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_predictions </span><span class="token operator">=</span><span class="token plain"> initial_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">initial_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">initial_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_predictions </span><span class="token operator">=</span><span class="token plain"> final_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">final_model_test_metric </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_model_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;R^2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;On the test data, the initial (untuned) model achieved R^2 </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">initial_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, and the final (tuned) model achieved </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">final_model_test_metric</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Random Search</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#requirements" class="table-of-contents__link toc-highlight">Requirements</a></li><li><a href="#mlflow-autologging" class="table-of-contents__link toc-highlight">MLflow autologging</a><ul><li><a href="#set-experiment-name-for-tracking" class="table-of-contents__link toc-highlight">Set experiment name for tracking</a></li></ul></li><li><a href="#part-1-run-distributed-training-using-mllib" class="table-of-contents__link toc-highlight">Part 1. Run distributed training using MLlib</a></li><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a><ul><li><a href="#create-a-function-to-train-a-model" class="table-of-contents__link toc-highlight">Create a function to train a model</a></li></ul></li><li><a href="#part-2-use-hyperopt-to-tune-hyperparameters" class="table-of-contents__link toc-highlight">Part 2. Use Hyperopt to tune hyperparameters</a><ul><li><a href="#define-a-function-to-minimize" class="table-of-contents__link toc-highlight">Define a function to minimize</a></li><li><a href="#define-the-search-space-over-hyperparameters" class="table-of-contents__link toc-highlight">Define the search space over hyperparameters</a></li><li><a href="#tune-the-model-using-hyperopt-fmin" class="table-of-contents__link toc-highlight">Tune the model using Hyperopt <code>fmin()</code></a></li><li><a href="#retrain-the-model-on-the-full-training-dataset" class="table-of-contents__link toc-highlight">Retrain the model on the full training dataset</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html b/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html
index 7c4fa08577..2314662895 100644
--- a/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
+++ b/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">HyperOpt</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">Quickstart - Random Search</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Hyperparameter Tuning</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Random Search</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>HyperParameterTuning - Fighting Breast Cancer</h1><p>This tutorial shows how SynapseML can be used to identify the best combination of hyperparameters for your chosen classifiers, ultimately resulting in more accurate and reliable models. In order to demonstrate this, we&#x27;ll show how to perform distributed randomized grid search hyperparameter tuning to build a model to identify breast cancer. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>Start by importing pandas and setting up our Spark session.</p><p>Next, read the data and split it into tuning and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BreastCancer.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Define the models to be used.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> TuneHyperparameters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---find-the-best-model-using-automl">2 - Find the best model using AutoML<a href="#2---find-the-best-model-using-automl" class="hash-link" aria-label="Direct link to 2 - Find the best model using AutoML" title="Direct link to 2 - Find the best model using AutoML">​</a></h2><p>Import SynapseML&#x27;s AutoML classes from <code>synapse.ml.automl</code>.
 Specify the hyperparameters using the <code>HyperparamBuilder</code>. Add either <code>DiscreteHyperParam</code> or <code>RangeHyperParam</code> hyperparameters. <code>TuneHyperparameters</code> will randomly choose values from a uniform distribution:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next, run TuneHyperparameters to get the best model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tune</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---evaluate-the-model">3 - Evaluate the model<a href="#3---evaluate-the-model" class="hash-link" aria-label="Direct link to 3 - Evaluate the model" title="Direct link to 3 - Evaluate the model">​</a></h2><p>We can view the best model&#x27;s parameters and retrieve the underlying best model pipeline</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModelInfo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can score against the test set and view metrics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">HyperOpt</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Transformations</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---find-the-best-model-using-automl" class="table-of-contents__link toc-highlight">2 - Find the best model using AutoML</a></li><li><a href="#3---evaluate-the-model" class="table-of-contents__link toc-highlight">3 - Evaluate the model</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/LightGBM/Overview/index.html b/docs/next/Explore Algorithms/LightGBM/Overview/index.html
index 7ed4984614..ce7773f768 100644
--- a/docs/next/Explore Algorithms/LightGBM/Overview/index.html	
+++ b/docs/next/Explore Algorithms/LightGBM/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -105,7 +105,7 @@
 to use Apache Spark&#x27;s <code>barrier()</code> stage to ensure all tasks execute at the same time.
 Barrier execution mode changes the logic to aggregate <code>host:port</code> information across all tasks in a synchronized way.
 To use it in scala, you can call setUseBarrierExecutionMode(true), for example:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbm = new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(labelColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setObjective(binaryObjective)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUseBarrierExecutionMode(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">&lt;train classifier&gt;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: barrier execution mode can also cause complicated issues, so use it only if needed.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Ranking, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li><li><a href="#advantages-of-lightgbm-through-synapseml" class="table-of-contents__link toc-highlight">Advantages of LightGBM through SynapseML</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#argumentsparameters" class="table-of-contents__link toc-highlight">Arguments/Parameters</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li><li><a href="#data-transfer-mode" class="table-of-contents__link toc-highlight">Data Transfer Mode</a></li><li><a href="#data-sampling" class="table-of-contents__link toc-highlight">Data Sampling</a></li><li><a href="#barrier-execution-mode" class="table-of-contents__link toc-highlight">Barrier Execution Mode</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html b/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html
index 02c7032d69..66f6301176 100644
--- a/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
+++ b/docs/next/Explore Algorithms/LightGBM/Quickstart - Classification, Ranking, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 parameters</a>,
 that one can use to customize their decision tree system. LightGBM on
 Spark also supports new types of problems such as quantile regression.</li><li><strong>Cross platform</strong>: LightGBM on Spark is available on Spark, PySpark, and SparklyR.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm-usage">LightGBM Usage<a href="#lightgbm-usage" class="hash-link" aria-label="Direct link to LightGBM Usage" title="Direct link to LightGBM Usage">​</a></h3><ul><li><strong>LightGBMClassifier</strong>: used for building classification models. For example, to predict whether a company bankrupts or not, we could build a binary classification model with <code>LightGBMClassifier</code>.</li><li><strong>LightGBMRegressor</strong>: used for building regression models. For example, to predict housing price, we could build a regression model with <code>LightGBMRegressor</code>.</li><li><strong>LightGBMRanker</strong>: used for building ranking models. For example, to predict the relevance of website search results, we could build a ranking model with <code>LightGBMRanker</code>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbmclassifier-to-train-a-classification-model">Use <code>LightGBMClassifier</code> to train a classification model<a href="#use-lightgbmclassifier-to-train-a-classification-model" class="hash-link" aria-label="Direct link to use-lightgbmclassifier-to-train-a-classification-model" title="Direct link to use-lightgbmclassifier-to-train-a-classification-model">​</a></h2><p>In this example, we use LightGBM to build a classification model in order to predict bankruptcy.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/company_bankruptcy_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset size</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test-sets">Split the dataset into train and test sets<a href="#split-the-dataset-into-train-and-test-sets" class="hash-link" aria-label="Direct link to Split the dataset into train and test sets" title="Direct link to Split the dataset into train and test sets">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="add-a-featurizer-to-convert-features-into-vectors">Add a featurizer to convert features into vectors<a href="#add-a-featurizer-to-convert-features-into-vectors" class="hash-link" aria-label="Direct link to Add a featurizer to convert features into vectors" title="Direct link to Add a featurizer to convert features into vectors">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="check-if-the-data-is-unbalanced">Check if the data is unbalanced<a href="#check-if-the-data-is-unbalanced" class="hash-link" aria-label="Direct link to Check if the data is unbalanced" title="Direct link to Check if the data is unbalanced">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> isUnbalance</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>&quot;saveNativeModel&quot; allows you to extract the underlying lightGBM model for fast deployment after you train on Spark.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMClassificationModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_synapse_internal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Files/models/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">saveNativeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model </span><span class="token operator">=</span><span class="token plain"> LightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loadNativeModelFromFile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/tmp/lgbmclassifier.model&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-importance">Visualize feature importance<a href="#visualize-feature-importance" class="hash-link" aria-label="Direct link to Visualize feature importance" title="Direct link to Visualize feature importance">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_importances </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Series</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_importances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> index</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fi </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ascending</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_index </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f_values </span><span class="token operator">=</span><span class="token plain"> fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print feature importances</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_index:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f_values:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># plot</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> x </span><span class="token keyword">in</span><span class="token plain"> x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rcParams</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;figure.figsize&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">barh</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    x_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.028</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> align</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tick_label</span><span class="token operator">=</span><span class="token plain">f_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generate-predictions-with-the-model">Generate predictions with the model<a href="#generate-predictions-with-the-model" class="hash-link" aria-label="Direct link to Generate predictions with the model" title="Direct link to Generate predictions with the model">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bankrupt?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbmregressor-to-train-a-quantile-regression-model">Use <code>LightGBMRegressor</code> to train a quantile regression model<a href="#use-lightgbmregressor-to-train-a-quantile-regression-model" class="hash-link" aria-label="Direct link to use-lightgbmregressor-to-train-a-quantile-regression-model" title="Direct link to use-lightgbmregressor-to-train-a-quantile-regression-model">​</a></h2><p>In this example, we show how to use LightGBM to build a regression model.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test-sets">Split dataset into train and test sets<a href="#split-dataset-into-train-and-test-sets" class="hash-link" aria-label="Direct link to Split dataset into train and test sets" title="Direct link to Split dataset into train and test sets">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="train-the-model-using-lightgbmregressor">Train the model using <code>LightGBMRegressor</code><a href="#train-the-model-using-lightgbmregressor" class="hash-link" aria-label="Direct link to train-the-model-using-lightgbmregressor" title="Direct link to train-the-model-using-lightgbmregressor">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getFeatureImportances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generate-predictions-with-the-model-1">Generate predictions with the model<a href="#generate-predictions-with-the-model-1" class="hash-link" aria-label="Direct link to Generate predictions with the model" title="Direct link to Generate predictions with the model">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-lightgbmranker-to-train-a-ranking-model">Use <code>LightGBMRanker</code> to train a ranking model<a href="#use-lightgbmranker-to-train-a-ranking-model" class="hash-link" aria-label="Direct link to use-lightgbmranker-to-train-a-ranking-model" title="Direct link to use-lightgbmranker-to-train-a-ranking-model">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-the-dataset">Read the dataset<a href="#read-the-dataset" class="hash-link" aria-label="Direct link to Read the dataset" title="Direct link to Read the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_train.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="train-the-ranking-model-using-lightgbmranker">Train the ranking model using <code>LightGBMRanker</code>.<a href="#train-the-ranking-model-using-lightgbmranker" class="hash-link" aria-label="Direct link to train-the-ranking-model-using-lightgbmranker" title="Direct link to train-the-ranking-model-using-lightgbmranker">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRanker</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker </span><span class="token operator">=</span><span class="token plain"> LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token plain">features_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    groupCol</span><span class="token operator">=</span><span class="token plain">query_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;preds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    leafPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPreds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresShapCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;importances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    repartitionByGroupingColumn</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evalAt</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ndcg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbm_ranker_model </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generate-predictions-with-the-model-2">Generate predictions with the model<a href="#generate-predictions-with-the-model-2" class="hash-link" aria-label="Direct link to Generate predictions with the model" title="Direct link to Generate predictions with the model">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dt </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/lightGBMRanker_test.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> lgbm_ranker_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-lightgbm" class="table-of-contents__link toc-highlight">What is LightGBM</a><ul><li><a href="#advantages-of-lightgbm" class="table-of-contents__link toc-highlight">Advantages of LightGBM</a></li><li><a href="#lightgbm-usage" class="table-of-contents__link toc-highlight">LightGBM Usage</a></li></ul></li><li><a href="#use-lightgbmclassifier-to-train-a-classification-model" class="table-of-contents__link toc-highlight">Use <code>LightGBMClassifier</code> to train a classification model</a><ul><li><a href="#read-dataset" class="table-of-contents__link toc-highlight">Read dataset</a></li><li><a href="#split-the-dataset-into-train-and-test-sets" class="table-of-contents__link toc-highlight">Split the dataset into train and test sets</a></li><li><a href="#add-a-featurizer-to-convert-features-into-vectors" class="table-of-contents__link toc-highlight">Add a featurizer to convert features into vectors</a></li><li><a href="#check-if-the-data-is-unbalanced" class="table-of-contents__link toc-highlight">Check if the data is unbalanced</a></li><li><a href="#model-training" class="table-of-contents__link toc-highlight">Model Training</a></li><li><a href="#visualize-feature-importance" class="table-of-contents__link toc-highlight">Visualize feature importance</a></li><li><a href="#generate-predictions-with-the-model" class="table-of-contents__link toc-highlight">Generate predictions with the model</a></li></ul></li><li><a href="#use-lightgbmregressor-to-train-a-quantile-regression-model" class="table-of-contents__link toc-highlight">Use <code>LightGBMRegressor</code> to train a quantile regression model</a><ul><li><a href="#read-dataset-1" class="table-of-contents__link toc-highlight">Read dataset</a></li><li><a href="#split-dataset-into-train-and-test-sets" class="table-of-contents__link toc-highlight">Split dataset into train and test sets</a></li><li><a href="#train-the-model-using-lightgbmregressor" class="table-of-contents__link toc-highlight">Train the model using <code>LightGBMRegressor</code></a></li><li><a href="#generate-predictions-with-the-model-1" class="table-of-contents__link toc-highlight">Generate predictions with the model</a></li></ul></li><li><a href="#use-lightgbmranker-to-train-a-ranking-model" class="table-of-contents__link toc-highlight">Use <code>LightGBMRanker</code> to train a ranking model</a><ul><li><a href="#read-the-dataset" class="table-of-contents__link toc-highlight">Read the dataset</a></li><li><a href="#train-the-ranking-model-using-lightgbmranker" class="table-of-contents__link toc-highlight">Train the ranking model using <code>LightGBMRanker</code>.</a></li><li><a href="#generate-predictions-with-the-model-2" class="table-of-contents__link toc-highlight">Generate predictions with the model</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/Langchain/index.html b/docs/next/Explore Algorithms/OpenAI/Langchain/index.html
index d45a0a58f4..295ee940b6 100644
--- a/docs/next/Explore Algorithms/OpenAI/Langchain/index.html	
+++ b/docs/next/Explore Algorithms/OpenAI/Langchain/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
-<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Langchain/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Langchain/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Langchain/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Langchain/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Langchain/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Langchain/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Langchain</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Using the LangChain Transformer</h1><p>LangChain is a software development framework designed to simplify the creation of applications using large language models (LLMs). Chains in LangChain go beyond just a single LLM call and are sequences of calls (can be a call to an LLM or a different utility), automating the execution of a series of calls and actions.
 To make it easier to scale up the LangChain execution on a large dataset, we have integrated LangChain with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of data with the LangChain Framework.</p><p>This tutorial shows how to apply LangChain at scale for paper summarization and organization. We start with a table of arxiv links and apply the LangChain Transformerto automatically extract the corresponding paper title, authors, summary, and some related works.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li>Import the notebook into <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a>, <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">Synapse Workspace</a> or if using Databricks into the <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">Databricks Workspace</a>.</li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported.</li><li>Connect your notebook to a cluster and follow along, editing and running the cells below.</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> langchain</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token number" style="color:rgb(174, 129, 255)">.331</span><span class="token plain"> pdf2image pdfminer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">six unstructured</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.10</span><span class="token number" style="color:rgb(174, 129, 255)">.24</span><span class="token plain"> pytesseract numpy</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">1.22</span><span class="token number" style="color:rgb(174, 129, 255)">.4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> uuid</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">llms </span><span class="token keyword">import</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OpenAI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agents </span><span class="token keyword">import</span><span class="token plain"> load_tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> AgentType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">chains </span><span class="token keyword">import</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> SimpleSequentialChain</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">document_loaders </span><span class="token keyword">import</span><span class="token plain"> OnlinePDFLoader</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bing_search</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tool </span><span class="token keyword">import</span><span class="token plain"> BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BingSearchAPIWrapper</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> langchain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">prompts </span><span class="token keyword">import</span><span class="token plain"> PromptTemplate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">langchain </span><span class="token keyword">import</span><span class="token plain"> LangchainTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> find_secret</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-the-service-information-and-construct-the-llm">Step 3: Fill in the service information and construct the LLM<a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="hash-link" aria-label="Direct link to Step 3: Fill in the service information and construct the LLM" title="Direct link to Step 3: Fill in the service information and construct the LLM">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>model_name</code>, <code>deployment_name</code>, <code>openai_api_base</code>, and <code>open_api_key</code> variables to match those for your OpenAI service. Please feel free to replace <code>find_secret</code> with your key as follows</p><p><code>openai_api_key = &quot;99sj2w82o....&quot;</code></p><p><code>bing_subscription_key = &quot;...&quot;</code></p><p>Note that you also need to set up your Bing search to gain access to your <a href="https://learn.microsoft.com/en-us/bing/search-apis/bing-web-search/create-bing-search-service-resource" target="_blank" rel="noopener noreferrer">Bing Search subscription key</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://synapseml-openai-2.openai.azure.com/&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2022-12-01&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://api.bing.microsoft.com/v7.0/search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_subscription_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SUBSCRIPTION_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_subscription_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> bing_search_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_TYPE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_type</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_VERSION&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_version</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_BASE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_base</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;OPENAI_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> openai_api_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">llm </span><span class="token operator">=</span><span class="token plain"> AzureOpenAI</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    deployment_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model_name</span><span class="token operator">=</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    temperature</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-basic-usage-of-langchain-transformer">Step 4: Basic Usage of LangChain Transformer<a href="#step-4-basic-usage-of-langchain-transformer" class="hash-link" aria-label="Direct link to Step 4: Basic Usage of LangChain Transformer" title="Direct link to Step 4: Basic Usage of LangChain Transformer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-chain">Create a chain<a href="#create-a-chain" class="hash-link" aria-label="Direct link to Create a chain" title="Direct link to Create a chain">​</a></h3><p>We will start by demonstrating the basic usage with a simple chain that creates definitions for input words</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">copy_prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    template</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Define the following word: {technology}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">copy_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;definition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-and-apply-the-chain">Create a dataset and apply the chain<a href="#create-a-dataset-and-apply-the-chain" class="hash-link" aria-label="Direct link to Create a dataset and apply the chain" title="Direct link to Create a dataset and apply the chain">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># construction of test dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;docker&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;python&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;technology&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-and-load-the-langchain-transformer">Save and load the LangChain transformer<a href="#save-and-load-the-langchain-transformer" class="hash-link" aria-label="Direct link to Save and load the LangChain transformer" title="Direct link to Save and load the LangChain transformer">​</a></h3><p>LangChain Transformers can be saved and loaded. Note that LangChain serialization only works for chains that don&#x27;t have memory.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">temp_dir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tmp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exists</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mkdir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">path </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">temp_dir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;langchainTransformer&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">loaded </span><span class="token operator">=</span><span class="token plain"> LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">loaded</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-using-langchain-for-large-scale-literature-review">Step 5: Using LangChain for Large scale literature review<a href="#step-5-using-langchain-for-large-scale-literature-review" class="hash-link" aria-label="Direct link to Step 5: Using LangChain for Large scale literature review" title="Direct link to Step 5: Using LangChain for Large scale literature review">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-sequential-chain-for-paper-summarization">Create a Sequential Chain for paper summarization<a href="#create-a-sequential-chain-for-paper-summarization" class="hash-link" aria-label="Direct link to Create a Sequential Chain for paper summarization" title="Direct link to Create a Sequential Chain for paper summarization">​</a></h3><p>We will now construct a Sequential Chain for extracting structured information from an arxiv link. In particular, we will ask langchain to extract the title, author information, and a summary of the paper content. After that, we use a web search tool to find the recent papers written by the first author.</p><p>To summarize, our sequential chain contains the following steps:</p><ol><li><strong>Transform Chain</strong>: Extract Paper Content from arxiv Link <strong>=&gt;</strong></li><li><strong>LLMChain</strong>: Summarize the Paper, extract paper title and authors <strong>=&gt;</strong></li><li><strong>Transform Chain</strong>: to generate the prompt <strong>=&gt;</strong></li><li><strong>Agent with Web Search Tool</strong>: Use Web Search to find the recent papers by the first author</li></ol><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    arxiv_link </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    loader </span><span class="token operator">=</span><span class="token plain"> OnlinePDFLoader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">arxiv_link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pages </span><span class="token operator">=</span><span class="token plain"> loader</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_and_split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content </span><span class="token operator">+</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">page_content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">-</span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output </span><span class="token operator">=</span><span class="token plain"> inputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;find the paper title, author, summary in the paper description below, output them. After that, Use websearch to find out 3 recent papers of the first author in the author section below (first author is the first name separated by comma) and list the paper titles in bullet points: &lt;Paper Description Start&gt;\n&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> output</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;Paper Description End&gt;.&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_content_extraction_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">paper_content_extraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_summarizer_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;You are a paper summarizer, given the paper content, it is your job to summarize the     paper into a short summary, and extract authors and paper title from the paper content.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Here is the paper content:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">{paper_content}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Output:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">paper title, authors and summary.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt </span><span class="token operator">=</span><span class="token plain"> PromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> template</span><span class="token operator">=</span><span class="token plain">paper_summarizer_template</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summarize_chain </span><span class="token operator">=</span><span class="token plain"> LLMChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">llm</span><span class="token operator">=</span><span class="token plain">llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prompt</span><span class="token operator">=</span><span class="token plain">prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompt_generation_chain </span><span class="token operator">=</span><span class="token plain"> TransformChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    input_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_variables</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    transform</span><span class="token operator">=</span><span class="token plain">prompt_generation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing </span><span class="token operator">=</span><span class="token plain"> BingSearchAPIWrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tools </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">BingSearchRun</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">api_wrapper</span><span class="token operator">=</span><span class="token plain">bing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">web_search_agent </span><span class="token operator">=</span><span class="token plain"> initialize_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tools</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> llm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> agent</span><span class="token operator">=</span><span class="token plain">AgentType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ZERO_SHOT_REACT_DESCRIPTION</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> verbose</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sequential_chain </span><span class="token operator">=</span><span class="token plain"> SimpleSequentialChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chains</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        paper_content_extraction_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        summarize_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prompt_generation_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        web_search_agent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="apply-the-langchain-transformer-to-perform-this-workload-at-scale">Apply the LangChain transformer to perform this workload at scale<a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="hash-link" aria-label="Direct link to Apply the LangChain transformer to perform this workload at scale" title="Direct link to Apply the LangChain transformer to perform this workload at scale">​</a></h3><p>We can now use our chain at scale using the <code>LangchainTransformer</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2107.13586.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2101.00190.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2103.10385.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://arxiv.org/pdf/2110.07602.pdf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># construct langchain transformer using the paper summarizer chain define above</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paper_info_extractor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    LangchainTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;arxiv_link&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paper_info&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sequential_chain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_api_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># extract paper information from arxiv links, the paper information needs to include:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># paper title, paper authors, brief paper summary, and recent papers published by the first author</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_info_extractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paper_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Quickstart - Predictive Maintenance/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Predictive Maintenance</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">OpenAI</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-the-service-information-and-construct-the-llm" class="table-of-contents__link toc-highlight">Step 3: Fill in the service information and construct the LLM</a></li><li><a href="#step-4-basic-usage-of-langchain-transformer" class="table-of-contents__link toc-highlight">Step 4: Basic Usage of LangChain Transformer</a><ul><li><a href="#create-a-chain" class="table-of-contents__link toc-highlight">Create a chain</a></li><li><a href="#create-a-dataset-and-apply-the-chain" class="table-of-contents__link toc-highlight">Create a dataset and apply the chain</a></li><li><a href="#save-and-load-the-langchain-transformer" class="table-of-contents__link toc-highlight">Save and load the LangChain transformer</a></li></ul></li><li><a href="#step-5-using-langchain-for-large-scale-literature-review" class="table-of-contents__link toc-highlight">Step 5: Using LangChain for Large scale literature review</a><ul><li><a href="#create-a-sequential-chain-for-paper-summarization" class="table-of-contents__link toc-highlight">Create a Sequential Chain for paper summarization</a></li><li><a href="#apply-the-langchain-transformer-to-perform-this-workload-at-scale" class="table-of-contents__link toc-highlight">Apply the LangChain transformer to perform this workload at scale</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/index.html b/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/index.html
new file mode 100644
index 0000000000..7f8a2927c3
--- /dev/null
+++ b/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/index.html	
@@ -0,0 +1,28 @@
+<!doctype html>
+<html lang="en" dir="ltr" class="docs-wrapper docs-doc-page docs-version-current plugin-docs plugin-id-default docs-doc-id-Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU">
+<head>
+<meta charset="UTF-8">
+<meta name="generator" content="Docusaurus v2.4.1">
+<title data-rh="true">Quickstart - Custom Embeddings and Approximate KNN on GPU | SynapseML</title><meta data-rh="true" name="viewport" content="width=device-width,initial-scale=1"><meta data-rh="true" name="twitter:card" content="summary_large_image"><meta data-rh="true" property="og:image" content="https://microsoft.github.io/SynapseML/img/synapseml_og.jpg"><meta data-rh="true" name="twitter:image" content="https://microsoft.github.io/SynapseML/img/synapseml_og.jpg"><meta data-rh="true" property="og:url" content="https://microsoft.github.io/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/"><meta data-rh="true" name="docusaurus_locale" content="en"><meta data-rh="true" name="docsearch:language" content="en"><meta data-rh="true" name="docusaurus_version" content="current"><meta data-rh="true" name="docusaurus_tag" content="docs-default-current"><meta data-rh="true" name="docsearch:version" content="current"><meta data-rh="true" name="docsearch:docusaurus_tag" content="docs-default-current"><meta data-rh="true" property="og:title" content="Quickstart - Custom Embeddings and Approximate KNN on GPU | SynapseML"><meta data-rh="true" name="description" content="The demo extending existing Azure OpenAI based demo when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using multilingual E5 text embeddings and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using NVIDIA TensorRT and Spark Rapids ML. The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to previos CPU based demo"><meta data-rh="true" property="og:description" content="The demo extending existing Azure OpenAI based demo when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using multilingual E5 text embeddings and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using NVIDIA TensorRT and Spark Rapids ML. The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to previos CPU based demo"><link data-rh="true" rel="icon" href="/SynapseML/img/favicon.ico"><link data-rh="true" rel="canonical" href="https://microsoft.github.io/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/"><link data-rh="true" rel="alternate" href="https://microsoft.github.io/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/" hreflang="en"><link data-rh="true" rel="alternate" href="https://microsoft.github.io/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/" hreflang="x-default"><link data-rh="true" rel="preconnect" href="https://GBW8AA15RD-dsn.algolia.net" crossorigin="anonymous"><link rel="alternate" type="application/rss+xml" href="/SynapseML/blog/rss.xml" title="SynapseML RSS Feed">
+<link rel="alternate" type="application/atom+xml" href="/SynapseML/blog/atom.xml" title="SynapseML Atom Feed">
+<link rel="alternate" type="application/json" href="/SynapseML/blog/feed.json" title="SynapseML JSON Feed">
+
+<link rel="preconnect" href="https://www.google-analytics.com">
+<link rel="preconnect" href="https://www.googletagmanager.com">
+<script async src="https://www.googletagmanager.com/gtag/js?id=G-RWPE0183E8"></script>
+<script>function gtag(){dataLayer.push(arguments)}window.dataLayer=window.dataLayer||[],gtag("js",new Date),gtag("config","G-RWPE0183E8",{anonymize_ip:!0})</script>
+
+
+<link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
+
+<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
+</head>
+<body class="navigation-with-keyboard">
+<script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Custom Embeddings and Approximate KNN on GPU</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with local (per node) NVIDIA TensorRT accelerator and GPU based Aproximate Nearest Neighbor (ANN)</h1><p>The demo extending existing <a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN.ipynb" target="_blank" rel="noopener noreferrer">Azure OpenAI based demo</a> when encoding is processed by OpenAI requests and KNN was using GPU based brute force search. This tutorial shows how to perform fast local embeddings using <a href="https://arxiv.org/abs/2402.05672" target="_blank" rel="noopener noreferrer">multilingual E5 text embeddings</a> and fast aproximate Nearest Neighbor search using IVFFlat alcorithm. All tutorial stages accelerated by NVIDIA GPU using <a href="https://developer.nvidia.com/tensorrt" target="_blank" rel="noopener noreferrer">NVIDIA TensorRT</a> and <a href="https://github.com/NVIDIA/spark-rapids-ml" target="_blank" rel="noopener noreferrer">Spark Rapids ML</a>. The tutorial folder contains two benchmark notebooks to demonstrate advantages of the presented GPU based approach compare to <a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">previos CPU based demo</a></p><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but currently the notebook was running on Databricks GPU based cluster using Standard_NC24ads_A100_v4 with 6 workers. Databricks Runtime was 13.3 LTS ML (includes Apache Spark 3.4.1, GPU, Scala 2.12) with related <a href="https://github.com/microsoft/SynapseML/tree/master/tools/init_scripts" target="_blank" rel="noopener noreferrer">init_script</a> to install all required packages.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prepare-environment">Step 1: Prepare Environment<a href="#step-1-prepare-environment" class="hash-link" aria-label="Direct link to Step 1: Prepare Environment" title="Direct link to Step 1: Prepare Environment">​</a></h2><p>It will imports required libraries and get initial settings</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> torch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> sys</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> VectorUDT</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> spark_rapids_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">knn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ApproximateNearestNeighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ApproximateNearestNeighborsModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hf </span><span class="token keyword">import</span><span class="token plain"> HuggingFaceSentenceEmbedder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> KNN</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-load-input-data">Step 2: Load Input Data<a href="#step-2-load-input-data" class="hash-link" aria-label="Direct link to Step 2: Load Input Data" title="Direct link to Step 2: Load Input Data">​</a></h2><p>It will load public dataset and generate extra syntetic rows if set by size parameter</p><p>The loaded dataset has 1000 rows. If you specify <i>number_of_input_rows</i> in <!-- -->[1..1000]<!-- --> it will cut extra rows if needed</p><p>If <i>number_of_input_rows</i> in <!-- -->[1000..1000000]<!-- --> it will generate extra rows using cross join of original data</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">file_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">file_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Size of DF</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">number_of_input_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Check if the row count is less than 10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> number_of_input_rows </span><span class="token operator">&lt;=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"> </span><span class="token keyword">or</span><span class="token plain"> number_of_input_rows </span><span class="token operator">&gt;=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">raise</span><span class="token plain"> ValueError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Limit is </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">number_of_input_rows</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">, which should be less than 1M.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> number_of_input_rows </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Cross-join the DataFrame with itself to create n x n pairs for string concatenation (synthetic data)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cross_joined_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crossJoin</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a new column &#x27;result_vector&#x27; by concatenating the two source vectors</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tmp_df </span><span class="token operator">=</span><span class="token plain"> cross_joined_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result_vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;. \n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Select only the necessary columns and show the result</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tmp_df </span><span class="token operator">=</span><span class="token plain"> tmp_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result_vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Shuffle the DataFrame with a fixed seed to have close strings spreaded</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    seed </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        tmp_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result_vector&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">number_of_input_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;Loaded: </span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">number_of_input_rows</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"> rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-generate-embeddings">Step 3: Generate Embeddings<a href="#step-3-generate-embeddings" class="hash-link" aria-label="Direct link to Step 3: Generate Embeddings" title="Direct link to Step 3: Generate Embeddings">​</a></h2><p>We will first generate embeddings using NVIDIA TensorRT optimized SentenceTransformer. In the demo you can use two fifferent HF models: intfloat/e5-large-v2 or sentence-transformers/all-MiniLM-L6-v2&quot;</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># To create embedder with different models, uncomment the following line</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># embedder = HuggingFaceSentenceEmbedder(modelName=&quot;intfloat/e5-large-v2&quot;, inputCol=&quot;data&quot;, outputCol=&quot;embeddings&quot;, runtime=&quot;tensorrt&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedder </span><span class="token operator">=</span><span class="token plain"> HuggingFaceSentenceEmbedder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    modelName</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence-transformers/all-MiniLM-L6-v2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    runtime</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tensorrt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embeddings </span><span class="token operator">=</span><span class="token plain"> embedder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-build-the-query-against-embeddings">Step 4: Build the query against embeddings<a href="#step-4-build-the-query-against-embeddings" class="hash-link" aria-label="Direct link to Step 4: Build the query against embeddings" title="Direct link to Step 4: Build the query against embeddings">​</a></h2><p>Get query embeddings running standard SentenceTransformer just on the driver. Convert embedding results to a data frame</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">queries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create DataFrame directly from the data and schema</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nullable</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nullable</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_embeddings </span><span class="token operator">=</span><span class="token plain"> embedder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-build-a-fast-vector-index-to-over-review-embeddings">Step 5: Build a fast vector index to over review embeddings<a href="#step-5-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 5: Build a fast vector index to over review embeddings" title="Direct link to Step 5: Build a fast vector index to over review embeddings">​</a></h2><p>We will use fast NVIDIA Rapids indexer. This KNN implementation will work only on GPU. If you want to use CPU then switch to synapse.ml.nn CPU based KNN implementation</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">RUN_ON_GPU </span><span class="token operator">=</span><span class="token plain"> torch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cuda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">is_available</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> RUN_ON_GPU</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rapids_knn_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ApproximateNearestNeighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array_to_vector_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorUDT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_with_vectors </span><span class="token operator">=</span><span class="token plain"> embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_to_vector_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    knn_model </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_with_vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-find-top-k-nearest-neighbors-on-gpu">Step 6: Find top k Nearest Neighbors ON GPU<a href="#step-6-find-top-k-nearest-neighbors-on-gpu" class="hash-link" aria-label="Direct link to Step 6: Find top k Nearest Neighbors ON GPU" title="Direct link to Step 6: Find top k Nearest Neighbors ON GPU">​</a></h2><p>We will use fast ANN <a href="https://developer.nvidia.com/blog/accelerated-vector-search-approximating-with-rapids-raft-ivf-flat/" target="_blank" rel="noopener noreferrer">IVFFlat algorithm</a> from Rapids</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> RUN_ON_GPU</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rapids_knn_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">kneighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array_to_vector_udf </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorUDT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_with_vectors </span><span class="token operator">=</span><span class="token plain"> query_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_to_vector_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    knn_df </span><span class="token operator">=</span><span class="token plain"> knn_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_with_vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-collect-and-display-results">Step 7: Collect and display results<a href="#step-7-collect-and-display-results" class="hash-link" aria-label="Direct link to Step 7: Collect and display results" title="Direct link to Step 7: Collect and display results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">if</span><span class="token plain"> RUN_ON_GPU</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zipped&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arrays_zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;indices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;distances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zipped.indices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zipped.distances&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    knn_df </span><span class="token operator">=</span><span class="token plain"> knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;original_data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    result_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;original_data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Results</h1><p>The goal of this demo is to showcase two acceleration techniques: local (per node) embedding generation and approximate KNN. Compared to the original method, which relies on HTTP requests to the OpenAI model and CPU-based KNN. The new approach is significantly more scalable and provides substantial acceleration, especially for large input datasets.</p><p>This is the comparison dureation results on 10 T4 GPU nodes for both approaches:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/knn_comparison.png" alt="KNN Comparison" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prepare-environment" class="table-of-contents__link toc-highlight">Step 1: Prepare Environment</a></li><li><a href="#step-2-load-input-data" class="table-of-contents__link toc-highlight">Step 2: Load Input Data</a></li><li><a href="#step-3-generate-embeddings" class="table-of-contents__link toc-highlight">Step 3: Generate Embeddings</a></li><li><a href="#step-4-build-the-query-against-embeddings" class="table-of-contents__link toc-highlight">Step 4: Build the query against embeddings</a></li><li><a href="#step-5-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 5: Build a fast vector index to over review embeddings</a></li><li><a href="#step-6-find-top-k-nearest-neighbors-on-gpu" class="table-of-contents__link toc-highlight">Step 6: Find top k Nearest Neighbors ON GPU</a></li><li><a href="#step-7-collect-and-display-results" class="table-of-contents__link toc-highlight">Step 7: Collect and display results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
+</body>
+</html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html b/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html
index 3c417eb706..e723df2ae6 100644
--- a/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html	
+++ b/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
-<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding and GPU based KNN</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI and GPU based KNN</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://github.com/NVIDIA/spark-rapids-ml/" target="_blank" rel="noopener noreferrer">Spark Rapids ML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on &quot;Quickstart - OpenAI Embedding&quot; notebook with NVIDIA GPU accelerated KNN.</p><p><strong>Note</strong>: Running the notebook with the demo dataset (Step 4) will generate the same results as CPU based “Quickstart - OpenAI Embedding” notebook. To see  GPU acceleration you need to run query against bigger embeddings.
-For example, running 100K rows dataset will give 6x acceleration and consume less than 10x memory on 2 nodes NVIDIA T4 cluster compare to AMD Epic (Rome) 2 nodes CPU cluster.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>To run the notebook on Databricks add related init script (/tools/init_scripts/init-rapidsml-cuda-11.8.sh)</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-query-against-embeddings">Step 8: Build the query against embeddings<a href="#step-8-build-the-query-against-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build the query against embeddings" title="Direct link to Step 8: Build the query against embeddings">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-fit-knn-model">Step 9: Fit KNN model<a href="#step-9-fit-knn-model" class="hash-link" aria-label="Direct link to Step 9: Fit KNN model" title="Direct link to Step 9: Fit KNN model">​</a></h2><p>Build KNN model using fit method</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> spark_rapids_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">knn </span><span class="token keyword">import</span><span class="token plain"> NearestNeighbors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn </span><span class="token operator">=</span><span class="token plain"> NearestNeighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn_model </span><span class="token operator">=</span><span class="token plain"> rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-10-retrieve-query-results">Step 10: Retrieve query results<a href="#step-10-retrieve-query-results" class="hash-link" aria-label="Direct link to Step 10: Retrieve query results" title="Direct link to Step 10: Retrieve query results">​</a></h2><p>Find k nearest neighbors</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rapids_knn_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">kneighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding_query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Understand and Search Forms</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-the-query-against-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build the query against embeddings</a></li><li><a href="#step-9-fit-knn-model" class="table-of-contents__link toc-highlight">Step 9: Fit KNN model</a></li><li><a href="#step-10-retrieve-query-results" class="table-of-contents__link toc-highlight">Step 10: Retrieve query results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding and GPU based KNN</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI and GPU based KNN</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://github.com/NVIDIA/spark-rapids-ml/" target="_blank" rel="noopener noreferrer">Spark Rapids ML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. This demo is based on &quot;Quickstart - OpenAI Embedding&quot; notebook with NVIDIA GPU accelerated KNN.</p><p><strong>Note</strong>: Running the notebook with the demo dataset (Step 4) will generate the same results as CPU based “Quickstart - OpenAI Embedding” notebook. To see  GPU acceleration you need to run query against bigger embeddings.
+For example, running 100K rows dataset will give 6x acceleration and consume less than 10x memory on 2 nodes NVIDIA T4 cluster compare to AMD Epic (Rome) 2 nodes CPU cluster.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>To run the notebook on Databricks add related init script (/tools/init_scripts/init-rapidsml-cuda-11.8.sh)</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-query-against-embeddings">Step 8: Build the query against embeddings<a href="#step-8-build-the-query-against-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build the query against embeddings" title="Direct link to Step 8: Build the query against embeddings">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-fit-knn-model">Step 9: Fit KNN model<a href="#step-9-fit-knn-model" class="hash-link" aria-label="Direct link to Step 9: Fit KNN model" title="Direct link to Step 9: Fit KNN model">​</a></h2><p>Build KNN model using fit method</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> spark_rapids_ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">knn </span><span class="token keyword">import</span><span class="token plain"> NearestNeighbors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn </span><span class="token operator">=</span><span class="token plain"> NearestNeighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">k</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rapids_knn_model </span><span class="token operator">=</span><span class="token plain"> rapids_knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-10-retrieve-query-results">Step 10: Retrieve query results<a href="#step-10-retrieve-query-results" class="hash-link" aria-label="Direct link to Step 10: Retrieve query results" title="Direct link to Step 10: Retrieve query results">​</a></h2><p>Find k nearest neighbors</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> _</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> rapids_knn_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">kneighbors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding_query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">knn_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Custom Embeddings and Approximate KNN on GPU</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-the-query-against-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build the query against embeddings</a></li><li><a href="#step-9-fit-knn-model" class="table-of-contents__link toc-highlight">Step 9: Fit KNN model</a></li><li><a href="#step-10-retrieve-query-results" class="table-of-contents__link toc-highlight">Step 10: Retrieve query results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html b/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html
index f3d71b164b..e6a099ea56 100644
--- a/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
+++ b/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
-<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - OpenAI Embedding</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Embedding Text with Azure OpenAI</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models to generate embeddings for large datasets of text. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-1-prerequisites">Step 1: Prerequisites<a href="#step-1-prerequisites" class="hash-link" aria-label="Direct link to Step 1: Prerequisites" title="Direct link to Step 1: Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-import-this-guide-as-a-notebook">Step 2: Import this guide as a notebook<a href="#step-2-import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Step 2: Import this guide as a notebook" title="Direct link to Step 2: Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ol><li><a href="https://github.com/microsoft/SynapseML/blob/master/notebooks/features/cognitive_services/CognitiveServices%20-%20OpenAI%20Embedding.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (click Raw, then save the file)</li><li>Import the notebook <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> or if using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Databricks Workspace</a></li><li>Install SynapseML on your cluster. Please see the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. Note that this requires pasting an additional cell at the top of the notebook you just imported</li><li>Connect your notebook to a cluster and follow along, editing and rnnung the cells below.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-fill-in-your-service-information">Step 3: Fill in your service information<a href="#step-3-fill-in-your-service-information" class="hash-link" aria-label="Direct link to Step 3: Fill in your service information" title="Direct link to Step 3: Fill in your service information">​</a></h2><p>Next, please edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match those for your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-load-data">Step 4: Load Data<a href="#step-4-load-data" class="hash-link" aria-label="Direct link to Step 4: Load Data" title="Direct link to Step 4: Load Data">​</a></h2><p>In this demo we will explore a dataset of fine food reviews</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">options</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inferSchema</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;True&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> delimiter</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> header</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">csv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/fine_food_reviews_1k.csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">format_string</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Title: %s; Content: %s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-5-generate-embeddings">Step 5: Generate Embeddings<a href="#step-5-generate-embeddings" class="hash-link" aria-label="Direct link to Step 5: Generate Embeddings" title="Direct link to Step 5: Generate Embeddings">​</a></h2><p>We will first generate embeddings for the reviews using the SynapseML OpenAIEmbedding client.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-6-reduce-embedding-dimensionality-for-visualization">Step 6: Reduce Embedding dimensionality for Visualization<a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="hash-link" aria-label="Direct link to Step 6: Reduce Embedding dimensionality for Visualization" title="Direct link to Step 6: Reduce Embedding dimensionality for Visualization">​</a></h2><p>We reduce the dimensionality to 2 dimensions using t-SNE decomposition.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">manifold </span><span class="token keyword">import</span><span class="token plain"> TSNE</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">collected </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">matrix </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">float64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scores </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> collected</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tsne </span><span class="token operator">=</span><span class="token plain"> TSNE</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n_components</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> perplexity</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random_state</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pca&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims </span><span class="token operator">=</span><span class="token plain"> tsne</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit_transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">matrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-7-plot-the-embeddings">Step 7: Plot the embeddings<a href="#step-7-plot-the-embeddings" class="hash-link" aria-label="Direct link to Step 7: Plot the embeddings" title="Direct link to Step 7: Plot the embeddings">​</a></h2><p>We now use t-SNE to reduce the dimensionality of the embeddings from 1536 to 2. Once the embeddings are reduced to two dimensions, we can plot them in a 2D scatter plot. We colour each review by its star rating, ranging from red for negative reviews, to green for positive reviews. We can observe a decent data separation even in the reduced 2 dimensions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colors </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkorange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gold&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;turquoise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;darkgreen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">x </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y </span><span class="token keyword">for</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y </span><span class="token keyword">in</span><span class="token plain"> vis_dims</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">color_indices </span><span class="token operator">=</span><span class="token plain"> scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">color_indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> score </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    avg_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">scores </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    color </span><span class="token operator">=</span><span class="token plain"> colors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">score</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">avg_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> avg_y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> marker</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;x&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token plain">color</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Amazon ratings visualized in language using t-SNE&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-a-fast-vector-index-to-over-review-embeddings">Step 8: Build a fast vector index to over review embeddings<a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="hash-link" aria-label="Direct link to Step 8: Build a fast vector index to over review embeddings" title="Direct link to Step 8: Build a fast vector index to over review embeddings">​</a></h2><p>We will use SynapseML&#x27;s KNN estimator to build a fast cosine-similarity retrieval engine.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn_index </span><span class="token operator">=</span><span class="token plain"> knn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-8-build-the-retrieval-model-pipeline">Step 8: Build the retrieval model pipeline<a href="#step-8-build-the-retrieval-model-pipeline" class="hash-link" aria-label="Direct link to Step 8: Build the retrieval model pipeline" title="Direct link to Step 8: Build the retrieval model pipeline">​</a></h2><p>Note: The data types of the ID columns in the document and query dataframes should be the same. For some OpenAI models, users should use separate models for embedding documents and queries. These models are denoted by the &quot;-doc&quot; and &quot;-query&quot; suffixes respectively.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> PipelineModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding_query </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">retrieval_model </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">embedding_query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> knn_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-9-retrieve-results">Step 9: Retrieve results<a href="#step-9-retrieve-results" class="hash-link" aria-label="Direct link to Step 9: Retrieve results" title="Direct link to Step 9: Retrieve results">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">query_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;desserts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;disgusting&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_matches </span><span class="token operator">=</span><span class="token plain"> retrieval_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;combined&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;match.distance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">OpenAI</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step-1-prerequisites" class="table-of-contents__link toc-highlight">Step 1: Prerequisites</a></li><li><a href="#step-2-import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Step 2: Import this guide as a notebook</a></li><li><a href="#step-3-fill-in-your-service-information" class="table-of-contents__link toc-highlight">Step 3: Fill in your service information</a></li><li><a href="#step-4-load-data" class="table-of-contents__link toc-highlight">Step 4: Load Data</a></li><li><a href="#step-5-generate-embeddings" class="table-of-contents__link toc-highlight">Step 5: Generate Embeddings</a></li><li><a href="#step-6-reduce-embedding-dimensionality-for-visualization" class="table-of-contents__link toc-highlight">Step 6: Reduce Embedding dimensionality for Visualization</a></li><li><a href="#step-7-plot-the-embeddings" class="table-of-contents__link toc-highlight">Step 7: Plot the embeddings</a></li><li><a href="#step-8-build-a-fast-vector-index-to-over-review-embeddings" class="table-of-contents__link toc-highlight">Step 8: Build a fast vector index to over review embeddings</a></li><li><a href="#step-8-build-the-retrieval-model-pipeline" class="table-of-contents__link toc-highlight">Step 8: Build the retrieval model pipeline</a></li><li><a href="#step-9-retrieve-results" class="table-of-contents__link toc-highlight">Step 9: Retrieve results</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html b/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html
index 67bca6ef2b..1cbfa6e115 100644
--- a/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
+++ b/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
-<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#form-recognizer" target="_blank" rel="noopener noreferrer">AnalyzeInvoices transformer</a> and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers (for Azure AI Document Intelligence) and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#translation" target="_blank" rel="noopener noreferrer">Translate</a>, a transformer that calls the Azure AI Translator service in Azure AI services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding and GPU based KNN</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Understand and Search Forms</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Tutorial: Create a custom search engine and question-answering system</h1><p>In this tutorial, learn how to index and query large data loaded from a Spark cluster. You set up a Jupyter Notebook that performs the following actions:</p><blockquote><ul><li>Load various forms (invoices) into a data frame in an Apache Spark session</li><li>Analyze them to determine their features</li><li>Assemble the resulting output into a tabular data structure</li><li>Write the output to a search index hosted in Azure Cognitive Search</li><li>Explore and query over the content you created</li></ul></blockquote><h2 class="anchor anchorWithStickyNavbar_LWe7" id="1---set-up-dependencies">1 - Set up dependencies<a href="#1---set-up-dependencies" class="hash-link" aria-label="Direct link to 1 - Set up dependencies" title="Direct link to 1 - Set up dependencies">​</a></h2><p>We start by importing packages and connecting to the Azure resources used in this workflow.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install openai</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">0.28</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string. e.g. cognitive_key=&quot;27snaiw...&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitive_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translator_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_service </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_index </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;form-demo-index-5&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;https://</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">openai_service_name</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">.openai.azure.com/&quot;</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="2---load-data-into-spark">2 - Load data into Spark<a href="#2---load-data-into-spark" class="hash-link" aria-label="Direct link to 2 - Load data into Spark" title="Direct link to 2 - Load data into Spark">​</a></h2><p>This code loads a few external files from an Azure storage account that&#x27;s used for demo purposes. The files are various invoices, and they&#x27;re read into a data frame.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> blob</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;@&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    container </span><span class="token operator">=</span><span class="token plain"> prefix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    split_postfix </span><span class="token operator">=</span><span class="token plain"> postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    account </span><span class="token operator">=</span><span class="token plain"> split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    filepath </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split_postfix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}/{}/{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">account</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> container</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filepath</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binaryFile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/form_subset/*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">blob_to_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;path&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Invoice11205.svg" width="40%" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="3---apply-form-recognition">3 - Apply form recognition<a href="#3---apply-form-recognition" class="hash-link" aria-label="Direct link to 3 - Apply form recognition" title="Direct link to 3 - Apply form recognition">​</a></h2><p>This code loads the <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#form-recognizer" target="_blank" rel="noopener noreferrer">AnalyzeInvoices transformer</a> and passes a reference to the data frame containing the invoices. It calls the pre-built invoice model of Azure Forms Analyzer.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzed_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitive_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="4---simplify-form-recognition-output">4 - Simplify form recognition output<a href="#4---simplify-form-recognition-output" class="hash-link" aria-label="Direct link to 4 - Simplify form recognition output" title="Direct link to 4 - Simplify form recognition output">​</a></h2><p>This code uses the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.services.form.html#module-synapse.ml.services.form.FormOntologyTransformer" target="_blank" rel="noopener noreferrer">FormOntologyLearner</a>, a transformer that analyzes the output of Form Recognizer transformers (for Azure AI Document Intelligence) and infers a tabular data structure. The output of AnalyzeInvoices is dynamic and varies based on the features detected in your content.</p><p>FormOntologyLearner extends the utility of the AnalyzeInvoices transformer by looking for patterns that can be used to create a tabular data structure. Organizing the output into multiple columns and rows makes for simpler downstream analysis.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">form </span><span class="token keyword">import</span><span class="token plain"> FormOntologyLearner</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">organized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FormOntologyLearner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;extracted.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>With our nice tabular dataframe, we can flatten the nested tables found in the forms with some SparkSQL</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itemized_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    organized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Items&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Item&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="5---add-translations">5 - Add translations<a href="#5---add-translations" class="hash-link" aria-label="Direct link to 5 - Add translations" title="Direct link to 5 - Add translations">​</a></h2><p>This code loads <a href="https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/#translation" target="_blank" rel="noopener noreferrer">Translate</a>, a transformer that calls the Azure AI Translator service in Azure AI services. The original text, which is in English in the &quot;Description&quot; column, is machine-translated into various languages. All of the output is consolidated into &quot;output.translations&quot; array.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">translate </span><span class="token keyword">import</span><span class="token plain"> Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translated_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translator_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ru&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">itemized_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TranslationError&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="6---translate-products-to-emojis-with-openai-">6 - Translate products to emojis with OpenAI 🤯<a href="#6---translate-products-to-emojis-with-openai-" class="hash-link" aria-label="Direct link to 6 - Translate products to emojis with OpenAI 🤯" title="Direct link to 6 - Translate products to emojis with OpenAI 🤯">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIPrompt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> split</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot; </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Your job is to translate item names into emoji. Do not add anything but the emoji and end the translation with a comma</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two Ducks: 🦆🦆,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Light Bulb: 💡,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Three Peaches: 🍑🍑🍑,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  Two kitchen stoves: ♨️♨️,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A red car: 🚗,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A person and a cat: 🧍🐈,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  A {Description}: &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prompter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIPrompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">openai_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emoji_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translated_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Emoji&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="7---infer-vendor-address-continent-with-openai">7 - Infer vendor address continent with OpenAI<a href="#7---infer-vendor-address-continent-with-openai" class="hash-link" aria-label="Direct link to 7 - Infer vendor address continent with OpenAI" title="Direct link to 7 - Infer vendor address continent with OpenAI">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_template </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Which continent does the following address belong to? </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Pick one value from Europe, Australia, North America, South America, Asia, Africa, Antarctica. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Dont respond with anything but one of the above. If you don&#x27;t know the answer or cannot figure it out from the text, return None. End your answer with a comma.</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd, North Whales&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: Europe,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;6693 Ryan Rd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent: None,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Address: &quot;{VendorAddress}&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">Continent:&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">continent_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prompter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptTemplate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_template</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emoji_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> trim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">split</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;VendorAddress&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Continent&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="8---create-an-azure-search-index-for-the-forms">8 - Create an Azure Search Index for the Forms<a href="#8---create-an-azure-search-index-for-the-forms" class="hash-link" aria-label="Direct link to 8 - Create an Azure Search Index for the Forms" title="Direct link to 8 - Create an Azure Search Index for the Forms">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> monotonically_increasing_id</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        subscriptionKey</span><span class="token operator">=</span><span class="token plain">search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SearchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        serviceName</span><span class="token operator">=</span><span class="token plain">search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        indexName</span><span class="token operator">=</span><span class="token plain">search_index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        keyCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DocID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="9---try-out-a-search-query">9 - Try out a search query<a href="#9---try-out-a-search-query" class="hash-link" aria-label="Direct link to 9 - Try out a search query" title="Direct link to 9 - Try out a search query">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">search_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://{}.search.windows.net/indexes/{}/docs/search?api-version=2019-05-06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_service</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> search_index</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;door&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="10---build-a-chatbot-that-can-use-azure-search-as-a-tool-">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧<a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="hash-link" aria-label="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧" title="Direct link to 10 - Build a chatbot that can use Azure Search as a tool 🧠🔧">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/notebooks/chatbot_flow_2.svg" width="40%" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> json</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> openai</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_type </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_base </span><span class="token operator">=</span><span class="token plain"> openai_url</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_key </span><span class="token operator">=</span><span class="token plain"> openai_key</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">api_version </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2023-03-15-preview&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_context_prompt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You are a chatbot designed to answer questions with the help of a search engine that has the following information:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">continent_df</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token string-interpolation interpolation">columns</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be brief. If you need to use the search engine to solve the please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Given the search engine above, what would you search for to answer the following question?</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Question: &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">question</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">Please output a json in the form of {{&quot;query&quot;: &quot;example_query&quot;}}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    search_results </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">post</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        search_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> json</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;search&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> headers</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">f&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You previously ran a search for &quot;</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">query</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot; which returned the following results:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string-interpolation interpolation">search_results</span><span class="token string-interpolation interpolation punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token string-interpolation string" style="color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">You should use the results to help you answer questions. If you dont know the answer to a question say &quot;I dont know&quot;. Do not lie or hallucinate information. Be Brief and mention which query you used to solve the problem. </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token string-interpolation string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    response </span><span class="token operator">=</span><span class="token plain"> openai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        engine</span><span class="token operator">=</span><span class="token plain">openai_deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> messages</span><span class="token operator">=</span><span class="token plain">messages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> max_tokens</span><span class="token operator">=</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> top_p</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.95</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;choices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;message&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">while</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            query </span><span class="token operator">=</span><span class="token plain"> json</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loads</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_query_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">return</span><span class="token plain"> prompt_gpt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> chat_context_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> search_result_prompt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;role&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> question</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> Exception </span><span class="token keyword">as</span><span class="token plain"> e</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">raise</span><span class="token plain"> e</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="11---asking-our-chatbot-a-question">11 - Asking our chatbot a question<a href="#11---asking-our-chatbot-a-question" class="hash-link" aria-label="Direct link to 11 - Asking our chatbot a question" title="Direct link to 11 - Asking our chatbot a question">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">custom_chatbot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;What did Luke Diaz buy?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="12---a-quick-double-check">12 - A quick double check<a href="#12---a-quick-double-check" class="hash-link" aria-label="Direct link to 12 - A quick double check" title="Direct link to 12 - A quick double check">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    continent_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;CustomerName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Luke Diaz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Custom Embeddings and Approximate KNN on GPU</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Getting Started</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#1---set-up-dependencies" class="table-of-contents__link toc-highlight">1 - Set up dependencies</a></li><li><a href="#2---load-data-into-spark" class="table-of-contents__link toc-highlight">2 - Load data into Spark</a></li><li><a href="#3---apply-form-recognition" class="table-of-contents__link toc-highlight">3 - Apply form recognition</a></li><li><a href="#4---simplify-form-recognition-output" class="table-of-contents__link toc-highlight">4 - Simplify form recognition output</a></li><li><a href="#5---add-translations" class="table-of-contents__link toc-highlight">5 - Add translations</a></li><li><a href="#6---translate-products-to-emojis-with-openai-" class="table-of-contents__link toc-highlight">6 - Translate products to emojis with OpenAI 🤯</a></li><li><a href="#7---infer-vendor-address-continent-with-openai" class="table-of-contents__link toc-highlight">7 - Infer vendor address continent with OpenAI</a></li><li><a href="#8---create-an-azure-search-index-for-the-forms" class="table-of-contents__link toc-highlight">8 - Create an Azure Search Index for the Forms</a></li><li><a href="#9---try-out-a-search-query" class="table-of-contents__link toc-highlight">9 - Try out a search query</a></li><li><a href="#10---build-a-chatbot-that-can-use-azure-search-as-a-tool-" class="table-of-contents__link toc-highlight">10 - Build a chatbot that can use Azure Search as a tool 🧠🔧</a></li><li><a href="#11---asking-our-chatbot-a-question" class="table-of-contents__link toc-highlight">11 - Asking our chatbot a question</a></li><li><a href="#12---a-quick-double-check" class="table-of-contents__link toc-highlight">12 - A quick double check</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenAI/index.html b/docs/next/Explore Algorithms/OpenAI/index.html
index 8c5604cf8e..27105fc524 100644
--- a/docs/next/Explore Algorithms/OpenAI/index.html	
+++ b/docs/next/Explore Algorithms/OpenAI/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
-<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure OpenAI. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="import-this-guide-as-a-notebook">Import this guide as a notebook<a href="#import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Import this guide as a notebook" title="Direct link to Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ul><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (select <strong>Raw</strong>, then save the file)</li><li>Import the notebook. <ul><li>If you are using Synapse Analytics <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> </li><li>If your are using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. </li><li>If you are using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li></ul></li><li>Install SynapseML on your cluster. See the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. <ul><li>If you are using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li></ul></li><li><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Connect your notebook to a cluster and follow along, editing and running the cells.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fill-in-service-information">Fill in service information<a href="#fill-in-service-information" class="hash-link" aria-label="Direct link to Fill in service information" title="Direct link to Fill in service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match them to your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-of-prompts">Create a dataset of prompts<a href="#create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Create a dataset of prompts" title="Direct link to Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-openaicompletion-apache-spark-client">Create the OpenAICompletion Apache Spark Client<a href="#create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Create the OpenAICompletion Apache Spark Client" title="Direct link to Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-the-dataframe-with-the-openaicompletion-client">Transform the dataframe with the OpenAICompletion Client<a href="#transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Transform the dataframe with the OpenAICompletion Client" title="Direct link to Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>After creating the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. Select just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. The completion text will be different from the sample.</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="more-usage-examples">More Usage Examples<a href="#more-usage-examples" class="hash-link" aria-label="Direct link to More Usage Examples" title="Direct link to More Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task.</p><p>For more information on using <code>OpenAIEmbedding</code> see our <a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
+<div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/OpenAI/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/OpenAI/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/OpenAI/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/OpenAI/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/OpenAI/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/OpenAI/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">Langchain</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/">OpenAI</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">Quickstart - OpenAI Embedding</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding and GPU based KNN/">Quickstart - OpenAI Embedding and GPU based KNN</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Custom Embeddings and Approximate KNN on GPU/">Quickstart - Custom Embeddings and Approximate KNN on GPU</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - Understand and Search Forms/">Quickstart - Understand and Search Forms</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/OpenAI/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">OpenAI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">OpenAI</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Azure OpenAI for big data</h1><p>The Azure OpenAI service can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library <a href="https://www.microsoft.com/en-us/research/blog/synapseml-a-simple-multilingual-and-massively-parallel-machine-learning-library/" target="_blank" rel="noopener noreferrer">SynapseML</a>. This integration makes it easy to use the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark</a> distributed computing framework to process millions of prompts with the OpenAI service. This tutorial shows how to apply large language models at a distributed scale using Azure OpenAI. </p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prerequisites">Prerequisites<a href="#prerequisites" class="hash-link" aria-label="Direct link to Prerequisites" title="Direct link to Prerequisites">​</a></h2><p>The key prerequisites for this quickstart include a working Azure OpenAI resource, and an Apache Spark cluster with SynapseML installed. We suggest creating a Synapse workspace, but an Azure Databricks, HDInsight, or Spark on Kubernetes, or even a python environment with the <code>pyspark</code> package will work. </p><ol><li>An Azure OpenAI resource – request access <a href="https://customervoice.microsoft.com/Pages/ResponsePage.aspx?id=v4j5cvGGr0GRqy180BHbR7en2Ais5pxKtso_Pz4b1_xUOFA5Qk1UWDRBMjg0WFhPMkIzTzhKQ1dWNyQlQCN0PWcu" target="_blank" rel="noopener noreferrer">here</a> before <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">creating a resource</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="import-this-guide-as-a-notebook">Import this guide as a notebook<a href="#import-this-guide-as-a-notebook" class="hash-link" aria-label="Direct link to Import this guide as a notebook" title="Direct link to Import this guide as a notebook">​</a></h2><p>The next step is to add this code into your Spark cluster. You can either create a notebook in your Spark platform and copy the code into this notebook to run the demo. Or download the notebook and import it into Synapse Analytics</p><ul><li><a href="https://github.com/microsoft/SynapseML/blob/master/docs/Explore%20Algorithms/OpenAI/OpenAI.ipynb" target="_blank" rel="noopener noreferrer">Download this demo as a notebook</a> (select <strong>Raw</strong>, then save the file)</li><li>Import the notebook. <ul><li>If you are using Synapse Analytics <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-development-using-notebooks#create-a-notebook" target="_blank" rel="noopener noreferrer">into the Synapse Workspace</a> </li><li>If your are using Databricks <a href="https://docs.microsoft.com/en-us/azure/databricks/notebooks/notebooks-manage#create-a-notebook" target="_blank" rel="noopener noreferrer">import into the Databricks Workspace</a>. </li><li>If you are using Fabric <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook" target="_blank" rel="noopener noreferrer">import into the Fabric Workspace</a></li></ul></li><li>Install SynapseML on your cluster. See the installation instructions for Synapse at the bottom of <a href="https://microsoft.github.io/SynapseML/" target="_blank" rel="noopener noreferrer">the SynapseML website</a>. <ul><li>If you are using Fabric, please check <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">Installation Guide</a>. This requires pasting an extra cell at the top of the notebook you imported. </li></ul></li><li><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">Connect your notebook to a cluster and follow along, editing and running the cells.</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="fill-in-service-information">Fill in service information<a href="#fill-in-service-information" class="hash-link" aria-label="Direct link to Fill in service information" title="Direct link to Fill in service information">​</a></h2><p>Next, edit the cell in the notebook to point to your service. In particular set the <code>service_name</code>, <code>deployment_name</code>, <code>location</code>, and <code>key</code> variables to match them to your OpenAI service:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Fill in the following lines with your service information</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Learn more about selecting which embedding model to choose: https://openai.com/blog/new-and-improved-embedding-model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml-openai-2&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gpt-35-turbo&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">deployment_name_embeddings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text-embedding-ada-002&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;openai-api-key-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># please replace this line with your key as a string</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">assert</span><span class="token plain"> key </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token plain"> </span><span class="token keyword">and</span><span class="token plain"> service_name </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-a-dataset-of-prompts">Create a dataset of prompts<a href="#create-a-dataset-of-prompts" class="hash-link" aria-label="Direct link to Create a dataset of prompts" title="Direct link to Create a dataset of prompts">​</a></h2><p>Next, create a dataframe consisting of a series of rows, with one prompt per row. </p><p>You can also load data directly from ADLS or other databases. For more information on loading and preparing Spark dataframes, see the <a href="https://spark.apache.org/docs/latest/sql-data-sources.html" target="_blank" rel="noopener noreferrer">Apache Spark data loading guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello my name is&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The best code is code thats&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML is &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-openaicompletion-apache-spark-client">Create the OpenAICompletion Apache Spark Client<a href="#create-the-openaicompletion-apache-spark-client" class="hash-link" aria-label="Direct link to Create the OpenAICompletion Apache Spark Client" title="Direct link to Create the OpenAICompletion Apache Spark Client">​</a></h2><p>To apply the OpenAI Completion service to your dataframe you created, create an OpenAICompletion object, which serves as a distributed client. Parameters of the service can be set either with a single value, or by a column of the dataframe with the appropriate setters on the <code>OpenAICompletion</code> object. Here we&#x27;re setting <code>maxTokens</code> to 200. A token is around four characters, and this limit applies to the sum of the prompt and the result. We&#x27;re also setting the <code>promptCol</code> parameter with the name of the prompt column in the dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAICompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="transform-the-dataframe-with-the-openaicompletion-client">Transform the dataframe with the OpenAICompletion Client<a href="#transform-the-dataframe-with-the-openaicompletion-client" class="hash-link" aria-label="Direct link to Transform the dataframe with the OpenAICompletion Client" title="Direct link to Transform the dataframe with the OpenAICompletion Client">​</a></h2><p>After creating the dataframe and the completion client, you can transform your input dataset and add a column called <code>completions</code> with all of the information the service adds. Select just the text for simplicity.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_df </span><span class="token operator">=</span><span class="token plain"> completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    completed_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions.choices.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your output should look something like this. The completion text will be different from the sample.</p><table><thead><tr><th align="center"><strong>prompt</strong></th><th align="center"><strong>error</strong></th><th align="center"><strong>text</strong></th></tr></thead><tbody><tr><td align="center">Hello my name is</td><td align="center">null</td><td align="center">Makaveli I&#x27;m eighteen years old and I want to   be a rapper when I grow up I love writing and making music I&#x27;m from Los   Angeles, CA</td></tr><tr><td align="center">The best code is code thats</td><td align="center">null</td><td align="center">understandable This is a subjective statement,   and there is no definitive answer.</td></tr><tr><td align="center">SynapseML is</td><td align="center">null</td><td align="center">A machine learning algorithm that is able to learn how to predict the future outcome of events.</td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="more-usage-examples">More Usage Examples<a href="#more-usage-examples" class="hash-link" aria-label="Direct link to More Usage Examples" title="Direct link to More Usage Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generating-text-embeddings">Generating Text Embeddings<a href="#generating-text-embeddings" class="hash-link" aria-label="Direct link to Generating Text Embeddings" title="Direct link to Generating Text Embeddings">​</a></h3><p>In addition to completing text, we can also embed text for use in downstream algorithms or vector retrieval architectures. Creating embeddings allows you to search and retrieve documents from large collections and can be used when prompt engineering isn&#x27;t sufficient for the task.</p><p>For more information on using <code>OpenAIEmbedding</code> see our <a href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/">embedding guide</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIEmbedding</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">embedding </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIEmbedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name_embeddings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;embeddings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">embedding</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="chat-completion">Chat Completion<a href="#chat-completion" class="hash-link" aria-label="Direct link to Chat Completion" title="Direct link to Chat Completion">​</a></h3><p>Models such as ChatGPT and GPT-4 are capable of understanding chats instead of single prompts. The <code>OpenAIChatCompletion</code> transformer exposes this functionality at scale.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">openai </span><span class="token keyword">import</span><span class="token plain"> OpenAIChatCompletion</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> Row</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">role</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> content</span><span class="token operator">=</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> name</span><span class="token operator">=</span><span class="token plain">role</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are an AI chatbot with red as your favorite color&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Whats your favorite color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;system&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;You are very excited&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                make_message</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;How are you today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">chat_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAIChatCompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMessagesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    chat_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">chat_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;messages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chat_completions.choices.message.content&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="improve-throughput-with-request-batching">Improve throughput with request batching<a href="#improve-throughput-with-request-batching" class="hash-link" aria-label="Direct link to Improve throughput with request batching" title="Direct link to Improve throughput with request batching">​</a></h3><p>The example makes several requests to the service, one for each prompt. To complete multiple prompts in a single request, use batch mode. First, in the OpenAICompletion object, instead of setting the Prompt column to &quot;Prompt&quot;, specify &quot;batchPrompt&quot; for the BatchPrompt column.
 To do so, create a dataframe with a list of prompts per row.</p><p>As of this writing there&#x27;s currently a limit of 20 prompts in a single request, and a hard limit of 2048 &quot;tokens&quot;, or approximately 1500 words.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The time has come&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Pleased to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Today stocks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Here&#x27;s to&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The only thing&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Ask not what&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Every litter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Next we create the OpenAICompletion object. Rather than setting the prompt column, set the batchPrompt column if your column is of type <code>Array[String]</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">batch_completion </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    OpenAICompletion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDeploymentName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">deployment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCustomServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxTokens</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchPromptCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;completions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>In the call to transform, a request will be made per row. Since there are multiple prompts in a single row, each request is sent with all prompts in that row. The results contain a row for each row in the request.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_batch_df </span><span class="token operator">=</span><span class="token plain"> batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_batch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="using-an-automatic-minibatcher">Using an automatic minibatcher<a href="#using-an-automatic-minibatcher" class="hash-link" aria-label="Direct link to Using an automatic minibatcher" title="Direct link to Using an automatic minibatcher">​</a></h3><p>If your data is in column format, you can transpose it to row format using SynapseML&#x27;s <code>FixedMiniBatcherTransformer</code>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> FixedMiniBatchTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">completed_autobatch_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">coalesce</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Force a single partition so that our little 4-row dataframe makes a batch of size 4, you can remove this step for large datasets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batchSize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;batchPrompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">batch_completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completed_autobatch_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-engineering-for-translation">Prompt engineering for translation<a href="#prompt-engineering-for-translation" class="hash-link" aria-label="Direct link to Prompt engineering for translation" title="Direct link to Prompt engineering for translation">​</a></h3><p>The Azure OpenAI service can solve many different natural language tasks through <a href="https://docs.microsoft.com/en-us/azure/cognitive-services/openai/how-to/completions" target="_blank" rel="noopener noreferrer">prompt engineering</a>. Here, we show an example of prompting for language translation:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Japanese: Ookina hako \nEnglish: Big box \nJapanese: Midori tako\nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;French: Quel heure et il au Montreal? \nEnglish: What time is it in Montreal? \nFrench: Ou est le poulet? \nEnglish:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="prompt-for-question-answering">Prompt for question answering<a href="#prompt-for-question-answering" class="hash-link" aria-label="Direct link to Prompt for question answering" title="Direct link to Prompt for question answering">​</a></h3><p>Here, we prompt GPT-3 for general-knowledge question answering:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">qa_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Q: Where is the Grand Canyon?\nA: The Grand Canyon is in Arizona.\n\nQ: What is the weight of the Burj Khalifa in kilograms?\nA:&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prompt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">completion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">qa_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Langchain</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Quickstart - OpenAI Embedding/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - OpenAI Embedding</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prerequisites" class="table-of-contents__link toc-highlight">Prerequisites</a></li><li><a href="#import-this-guide-as-a-notebook" class="table-of-contents__link toc-highlight">Import this guide as a notebook</a></li><li><a href="#fill-in-service-information" class="table-of-contents__link toc-highlight">Fill in service information</a></li><li><a href="#create-a-dataset-of-prompts" class="table-of-contents__link toc-highlight">Create a dataset of prompts</a></li><li><a href="#create-the-openaicompletion-apache-spark-client" class="table-of-contents__link toc-highlight">Create the OpenAICompletion Apache Spark Client</a></li><li><a href="#transform-the-dataframe-with-the-openaicompletion-client" class="table-of-contents__link toc-highlight">Transform the dataframe with the OpenAICompletion Client</a></li><li><a href="#more-usage-examples" class="table-of-contents__link toc-highlight">More Usage Examples</a><ul><li><a href="#generating-text-embeddings" class="table-of-contents__link toc-highlight">Generating Text Embeddings</a></li><li><a href="#chat-completion" class="table-of-contents__link toc-highlight">Chat Completion</a></li><li><a href="#improve-throughput-with-request-batching" class="table-of-contents__link toc-highlight">Improve throughput with request batching</a></li><li><a href="#using-an-automatic-minibatcher" class="table-of-contents__link toc-highlight">Using an automatic minibatcher</a></li><li><a href="#prompt-engineering-for-translation" class="table-of-contents__link toc-highlight">Prompt engineering for translation</a></li><li><a href="#prompt-for-question-answering" class="table-of-contents__link toc-highlight">Prompt for question answering</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/OpenCV/Image Transformations/index.html b/docs/next/Explore Algorithms/OpenCV/Image Transformations/index.html
index a594f831c7..feb2b5b046 100644
--- a/docs/next/Explore Algorithms/OpenCV/Image Transformations/index.html	
+++ b/docs/next/Explore Algorithms/OpenCV/Image Transformations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 Internally, operations are pipelined and backed by OpenCV implementation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># images are resized and then cropped</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">crop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small </span><span class="token operator">=</span><span class="token plain"> tr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take third image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For the advanced image manipulations, use Spark UDFs.
 The SynapseML package provides conversion function between <em>Spark Row</em> and
 <em>ndarray</em> image representations.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> toImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array </span><span class="token operator">=</span><span class="token plain"> toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># convert Image to numpy ndarray[height, width, 3]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> toImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># numpy array back to Spark Row structure</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noBlueUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">u</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ImageSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">noblue </span><span class="token operator">=</span><span class="token plain"> small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> noBlueUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># take second image</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">toNDArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># display the image inside notebook</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Images could be unrolled into the dense 1D vectors suitable for CNTK evaluation.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> UnrollImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroller </span><span class="token operator">=</span><span class="token plain"> UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noblue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unrolled </span><span class="token operator">=</span><span class="token plain"> unroller</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">noblue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unrolled&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vector </span><span class="token operator">=</span><span class="token plain"> unrolled</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/Quickstart - Random Search/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Random Search</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#opencv---pipeline-image-transformations" class="table-of-contents__link toc-highlight">OpenCV - Pipeline Image Transformations</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/index.html b/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/index.html
index 059a1a4c6c..e0eaea5d4c 100644
--- a/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
+++ b/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -72,7 +72,7 @@
 This transformer is used internally by AccessAnomaly to normalize a model to generate
 anomaly scores with mean 0.0 and standard deviation of 1.0.</li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cyber.anomaly.html#synapse.ml.cyber.anomaly.collaborative_filtering.AccessAnomalyConfig" target="_blank" rel="noopener noreferrer">AccessAnomalyConfig</a>
 contains the default values for AccessAnomaly.</li></ol></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">SAR Algorithm</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#access-anomalies-complement_accesspy" class="table-of-contents__link toc-highlight">access anomalies: complement_access.py</a></li><li><a href="#feature-engineering-indexerspy" class="table-of-contents__link toc-highlight">feature engineering: indexers.py</a></li><li><a href="#feature-engineering-scalerspy" class="table-of-contents__link toc-highlight">feature engineering: scalers.py</a></li><li><a href="#access-anomalies-collaborative_filteringpy" class="table-of-contents__link toc-highlight">access anomalies: collaborative_filtering.py</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html b/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html
index 611025c2d4..b4839fbf29 100644
--- a/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
+++ b/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -36,7 +36,7 @@
 This is what this example demonstrates.</p><p>Note: the data does NOT contain information about departments, this information is implicitly learned by the model by analyzing the access patterns.</p><h1>Create an Azure Databricks cluster and install the following libs</h1><ol><li><p>In Cluster Libraries install from library source Maven:
 Coordinates: com.microsoft.azure:synapseml_2.12:1.0.4
 Repository: <a href="https://mmlspark.azureedge.net/maven" target="_blank" rel="noopener noreferrer">https://mmlspark.azureedge.net/maven</a></p></li><li><p>In Cluster Libraries install from PyPI the library called plotly</p></li></ol><h1>Setup &amp; Initialization</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install plotly</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># this is used to produce the synthetic dataset for this test</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dataset </span><span class="token keyword">import</span><span class="token plain"> DataFactory</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cyber</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collaborative_filtering </span><span class="token keyword">import</span><span class="token plain"> AccessAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> f</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Load up datasets</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> running_on_synapse</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dbfs:/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparkContext</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCheckpointDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;./tmp/checkpoint_path/&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">factory </span><span class="token operator">=</span><span class="token plain"> DataFactory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_hr_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_fin_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_users</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_eng_resources</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    single_component</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_pdf </span><span class="token operator">=</span><span class="token plain"> factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_training_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratio</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># a tenant id is used when independent datasets originate from different tenants, in this example we set all tenants-ids to the same value</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_intra_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    factory</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_clustered_inter_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Model setup &amp; training</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">access_anomaly </span><span class="token operator">=</span><span class="token plain"> AccessAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tenantCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    userCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    resCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    likelihoodCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;likelihood&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> access_anomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Apply model &amp; show result stats</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ingroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ingroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">outgroup_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;min_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;max_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mean</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mean_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stddev</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;stddev_anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Examine results</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Select a subset of results to send to Log Analytics</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">full_res_df </span><span class="token operator">=</span><span class="token plain"> outgroup_scored_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">window </span><span class="token keyword">import</span><span class="token plain"> Window</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># select values above threshold</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_above_threshold </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">anomaly_score </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># get distinct resource/user and corresponding timestamp and highest score</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_above_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index == 1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add a fake timestamp to the results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_la </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">current_timestamp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h1>Display all resource accesses by users with highest anomalous score</h1><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly </span><span class="token keyword">import</span><span class="token plain"> __version__</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">offline </span><span class="token keyword">import</span><span class="token plain"> download_plotlyjs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> iplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">__version__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># requires version &gt;= 1.9.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># run plotly in offline mode</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">init_notebook_mode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Find all server accesses of users with high predicted scores</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># For display, limit to top 25 results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results_to_display </span><span class="token operator">=</span><span class="token plain"> results_to_la</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interesting_records </span><span class="token operator">=</span><span class="token plain"> full_res_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_semi&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> interesting_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left_anti&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_records </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick only a subset of non-anomalous record for UI</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">w </span><span class="token operator">=</span><span class="token plain"> Window</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">partitionBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># pick top non-anomalous set</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">top_non_anomalous_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_records</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">row_number</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">over</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index in (1,2,3,4,5)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># add back anomalous record</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fileShare_accesses </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    top_non_anomalous_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># get unique users and file shares</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_df </span><span class="token operator">=</span><span class="token plain"> fileShare_accesses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">user</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">res</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">unique</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token plain">unique_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># create index for source &amp; target and color for the normal accesses</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">normal_line_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rgba(211, 211, 211, 0.8)&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomolous_color </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;red&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">high_scores_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> normal_line_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># results_to_display index, color and</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> results_to_display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tenant_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly_score&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> left_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> right_on</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> anomolous_color</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># subtract 1 for the red entries in all_access df</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">hsi_df </span><span class="token operator">=</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">merge</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    hsi_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;left&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    left_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    right_on</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsiCount&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">-</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">loc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">&gt;</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">all_access_updated_count_df </span><span class="token operator">=</span><span class="token plain"> all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;user&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;res&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># combine the two tables</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">frames </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">all_access_updated_count_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> high_scores_index_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display_df </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">frames</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sort</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># display_df.head()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data_trace </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sankey&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    domain</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    orientation</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;h&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    valueformat</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.0f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    node</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        pad</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        thickness</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        line</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        label</span><span class="token operator">=</span><span class="token plain">unique_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    link</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        source</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;userIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        target</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        value</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;count&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color</span><span class="token operator">=</span><span class="token plain">display_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> how</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;any&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">layout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    title</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;All resources accessed by users with highest anomalous scores&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">772</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    font</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">data_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> layout</span><span class="token operator">=</span><span class="token plain">layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">p </span><span class="token operator">=</span><span class="token plain"> plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> output_type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;div&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">p</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">CyberML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html b/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html
index 67057ec966..6397050ac5 100644
--- a/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
+++ b/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">SAR Algorithm</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/">CyberML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/">Quickstart - Anomalous Access Detection</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">Quickstart - Exploring Art Across Cultures</a></li></ul></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Other Algorithms</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Exploring Art Across Cultures</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Exploring Art across Culture and Medium with Fast, Conditional, k-Nearest Neighbors</h1><p>This article serves as a guideline for match-finding via k-nearest-neighbors. You set up code that allows queries involving cultures and mediums of art amassed from the Metropolitan Museum of Art in NYC and the Rijksmuseum in Amsterdam.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overview-of-the-balltree">Overview of the BallTree<a href="#overview-of-the-balltree" class="hash-link" aria-label="Direct link to Overview of the BallTree" title="Direct link to Overview of the BallTree">​</a></h2><p>The structure functioning behind the KNN model is a BallTree, which is a recursive binary tree where each node (or &quot;ball&quot;) contains a partition of the points of data to be queried. Building a BallTree involves assigning data points to the &quot;ball&quot; whose center they&#x27;re closest to (with respect to a certain specified feature), resulting in a structure that allows binary-tree-like traversal and lends itself to finding k-nearest neighbors at a BallTree leaf.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="setup">Setup<a href="#setup" class="hash-link" aria-label="Direct link to Setup" title="Direct link to Setup">​</a></h2><p>Import necessary Python libraries and prepare dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array_contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ConditionalKNNModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> io </span><span class="token keyword">import</span><span class="token plain"> BytesIO</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Our dataset comes from a table containing artwork information from both the Met and Rijks museums. The schema is as follows:</p><ul><li><strong>id</strong>: A unique identifier for a piece of art<ul><li>Sample Met id: <em>388395</em> </li><li>Sample Rijks id: <em>SK-A-2344</em> </li></ul></li><li><strong>Title</strong>: Art piece title, as written in the museum&#x27;s database</li><li><strong>Artist</strong>: Art piece artist, as written in the museum&#x27;s database</li><li><strong>Thumbnail_Url</strong>: Location of a JPEG thumbnail of the art piece</li><li><strong>Image_Url</strong> Location of an image of the art piece hosted on the Met/Rijks website</li><li><strong>Culture</strong>: Category of culture that the art piece falls under<ul><li>Sample culture categories: <em>latin american</em>, <em>egyptian</em>, etc.</li></ul></li><li><strong>Classification</strong>: Category of medium that the art piece falls under<ul><li>Sample medium categories: <em>woodwork</em>, <em>paintings</em>, etc.</li></ul></li><li><strong>Museum_Page</strong>: Link to the work of art on the Met/Rijks website</li><li><strong>Norm_Features</strong>: Embedding of the art piece image</li><li><strong>Museum</strong>: Specifies which museum the piece originated from</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># loads the dataset and the two trained CKNN models for querying by medium and culture</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/met_and_rijks.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-categories-to-be-queried-on">Define categories to be queried on<a href="#define-categories-to-be-queried-on" class="hash-link" aria-label="Direct link to Define categories to be queried on" title="Direct link to Define categories to be queried on">​</a></h2><p>Two KNN models are used: one for culture, and one for medium.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># mediums = [&#x27;prints&#x27;, &#x27;drawings&#x27;, &#x27;ceramics&#x27;, &#x27;textiles&#x27;, &#x27;paintings&#x27;, &quot;musical instruments&quot;,&quot;glass&quot;, &#x27;accessories&#x27;, &#x27;photographs&#x27;,  &quot;metalwork&quot;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#           &quot;sculptures&quot;, &quot;weapons&quot;, &quot;stone&quot;, &quot;precious&quot;, &quot;paper&quot;, &quot;woodwork&quot;, &quot;leatherwork&quot;, &quot;uncategorized&quot;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mediums </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;paintings&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ceramics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># cultures = [&#x27;african (general)&#x27;, &#x27;american&#x27;, &#x27;ancient american&#x27;, &#x27;ancient asian&#x27;, &#x27;ancient european&#x27;, &#x27;ancient middle-eastern&#x27;, &#x27;asian (general)&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;austrian&#x27;, &#x27;belgian&#x27;, &#x27;british&#x27;, &#x27;chinese&#x27;, &#x27;czech&#x27;, &#x27;dutch&#x27;, &#x27;egyptian&#x27;]#, &#x27;european (general)&#x27;, &#x27;french&#x27;, &#x27;german&#x27;, &#x27;greek&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;iranian&#x27;, &#x27;italian&#x27;, &#x27;japanese&#x27;, &#x27;latin american&#x27;, &#x27;middle eastern&#x27;, &#x27;roman&#x27;, &#x27;russian&#x27;, &#x27;south asian&#x27;, &#x27;southeast asian&#x27;,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#            &#x27;spanish&#x27;, &#x27;swiss&#x27;, &#x27;various&#x27;]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cultures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;japanese&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;american&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;african (general)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Uncomment the above for more robust and large scale searches!</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">classes </span><span class="token operator">=</span><span class="token plain"> cultures </span><span class="token operator">+</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_set </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">selected_ids </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RBK-17525-2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-MAK-1204&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AK-RAK-2015-2-9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">lambda</span><span class="token plain"> medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> id_val</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium </span><span class="token keyword">in</span><span class="token plain"> medium_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture </span><span class="token keyword">in</span><span class="token plain"> culture_set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">or</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">id_val </span><span class="token keyword">in</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-and-fit-conditionalknn-models">Define and fit ConditionalKNN models<a href="#define-and-fit-conditionalknn-models" class="hash-link" aria-label="Direct link to Define and fit ConditionalKNN models" title="Direct link to Define and fit ConditionalKNN models">​</a></h2><p>Create ConditionalKNN models for both the medium and culture columns; each model takes in an output column, features column (feature vector), values column (cell values under the output column), and label column (the quality that the respective KNN is conditioned on).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">medium_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">culture_cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Norm_Features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setValuesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Culture&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="define-matching-and-visualizing-methods">Define matching and visualizing methods<a href="#define-matching-and-visualizing-methods" class="hash-link" aria-label="Direct link to Define matching and visualizing methods" title="Direct link to Define matching and visualizing methods">​</a></h2><p>After the initial dataset and category setup, prepare methods that will query and visualize the conditional KNN&#x27;s results.</p><p><code>addMatches()</code> creates a Dataframe with a handful of matches per category.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> classes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        results </span><span class="token operator">=</span><span class="token plain"> cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumnRenamed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><code>plot_urls()</code> calls <code>plot_img</code> to visualize top matches for each category into a grid.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        response </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">response</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> aspect</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> title </span><span class="token keyword">is</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fontsize</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">axis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;off&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">=</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ny </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># reshape required in the case of 1 image query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes </span><span class="token operator">=</span><span class="token plain"> axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ny</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> j </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> titles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                plot_img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">savefig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">filename</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dpi</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1600</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># saves the results as a PNG</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="putting-it-all-together">Putting it all together<a href="#putting-it-all-together" class="hash-link" aria-label="Direct link to Putting it all together" title="Direct link to Putting it all together">​</a></h2><p>Define <code>test_all()</code> to take in the data, CKNN models, the art id values to query on, and the file path to save the output visualization to. The medium and culture models were previously trained and loaded.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># main method to test a particular dataset with two CKNN models and a set of art IDs, saving the result to filename.png</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    is_nice_obj </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> obj </span><span class="token keyword">in</span><span class="token plain"> test_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    test_df </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">where</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">is_nice_obj</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_medium </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results_df_culture </span><span class="token operator">=</span><span class="token plain"> add_matches</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cknn_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> results_df_medium</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    results </span><span class="token operator">=</span><span class="token plain"> results_df_culture</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    original_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Thumbnail_Url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> cultures</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    culture_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> culture_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">culture_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> cultures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_culture.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_urls </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Matches_{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> label </span><span class="token keyword">in</span><span class="token plain"> mediums</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    medium_url_arr </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">original_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> medium_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_urls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">medium_url_arr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Original&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> mediums</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches_by_medium.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> results_df_culture</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="demo">Demo<a href="#demo" class="hash-link" aria-label="Direct link to Demo" title="Direct link to Demo">​</a></h2><p>The following cell performs batched queries given desired image IDs and a filename to save the visualization.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/art/cross_cultural_matches.jpg" width="600" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># sample query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result_df </span><span class="token operator">=</span><span class="token plain"> test_all</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">small_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> medium_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> culture_cknn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> selected_ids</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> root</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Anomalous Access Detection/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Anomalous Access Detection</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview-of-the-balltree" class="table-of-contents__link toc-highlight">Overview of the BallTree</a></li><li><a href="#setup" class="table-of-contents__link toc-highlight">Setup</a></li><li><a href="#define-categories-to-be-queried-on" class="table-of-contents__link toc-highlight">Define categories to be queried on</a></li><li><a href="#define-and-fit-conditionalknn-models" class="table-of-contents__link toc-highlight">Define and fit ConditionalKNN models</a></li><li><a href="#define-matching-and-visualizing-methods" class="table-of-contents__link toc-highlight">Define matching and visualizing methods</a></li><li><a href="#putting-it-all-together" class="table-of-contents__link toc-highlight">Putting it all together</a></li><li><a href="#demo" class="table-of-contents__link toc-highlight">Demo</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html b/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html
index 5990006e8a..505a10532a 100644
--- a/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
+++ b/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -34,7 +34,7 @@
 <em>+ sim(Item 4, Item 5) <!-- -->*<!-- --> aff(User 1, Item 5)</em>
 <em>= <strong>3 <!-- -->*<!-- --> 5</strong> + 2 <!-- -->*<!-- --> 3 + 3 <!-- -->*<!-- --> 2.5 + 4 <!-- -->*<!-- --> 0 + 2 <!-- -->*<!-- --> 0</em>
 <!-- -->*<!-- -->= <strong>15</strong> + 6 + 7.5 + 0 + 0 = <strong>28.5<!-- -->*</strong></p><p>Clearly, the first term (highlighted) has the highest contribution to the score. We can say that &quot;The algorithm recommends Item 4 to User 1 because it&#x27;s similar to Item 1, to which User 1 has high affinity&quot;. A message like this can be displayed automatically for each recommendation.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Cyber ML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">CyberML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#input" class="table-of-contents__link toc-highlight">Input</a></li><li><a href="#collaborative-filtering" class="table-of-contents__link toc-highlight">Collaborative Filtering</a></li><li><a href="#making-recommendations" class="table-of-contents__link toc-highlight">Making Recommendations</a><ul><li><a href="#user-recommendations" class="table-of-contents__link toc-highlight">User Recommendations</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html b/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html
index 561b6beeb1..262bb2a393 100644
--- a/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
+++ b/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -43,7 +43,7 @@
 (<code>l-bfgs</code>), an <code>ElasticNet</code> mixing parameter of <code>0.3</code>, and a <code>Regularization</code>
 of <code>0.1</code>.</p><p>Train the model with the <code>TrainRegressor</code> API fit on the training dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> TrainedRegressorModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testCat </span><span class="token operator">=</span><span class="token plain"> DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Carrier&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DepTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrTimeBlk&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> convertTo</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;toCategorical&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Score the regressor on the test data.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testCat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Compute model metrics against the entire scored dataset</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, compute and show statistics on individual predictions in the test
 dataset, demonstrating the usage of <code>ComputePerInstanceStatistics</code></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance </span><span class="token operator">=</span><span class="token plain"> ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ArrDelay&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Vowpal Wabbit on Text Data/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit on Text Data</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression----flight-delays-with-datacleaning" class="table-of-contents__link toc-highlight">Regression -  Flight Delays with DataCleaning</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html b/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html
index 1df520b150..a2f158c9d4 100644
--- a/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
+++ b/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -60,7 +60,7 @@
 to evaluate the predictions.  The following statistics are calculated for
 regression models to evaluate:</p><ul><li>Mean squared error</li><li>Root mean squared error</li><li>R^2</li><li>Mean absolute error</li></ul><p>Use the <code>ComputeModelStatistics</code> API to compute basic statistics for
 the Poisson and the Random Forest models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Poisson Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">poissonMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Random Forest Metrics&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestMetrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can also compute per instance statistics for <code>poissonPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputePerInstanceStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pred</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;price&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L1_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;L2_loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">poissonPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>and with <code>randomForestPrediction</code>:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">demonstrateEvalPerInstance</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomForestPrediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Cleaning</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Vowpal Wabbit and LightGBM</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#regression---auto-imports" class="table-of-contents__link toc-highlight">Regression - Auto Imports</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html b/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html
index 8537ff1902..5820e8391a 100644
--- a/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
+++ b/docs/next/Explore Algorithms/Regression/Quickstart - Vowpal Wabbit and LightGBM/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -28,7 +28,7 @@
 <a href="https://spark.apache.org/docs/latest/ml-classification-regression.html#linear-regression" target="_blank" rel="noopener noreferrer">Spark MLlib Linear Regression</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="prepare-dataset">Prepare Dataset<a href="#prepare-dataset" class="hash-link" aria-label="Direct link to Prepare Dataset" title="Direct link to Prepare Dataset">​</a></h2><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following is the summary of the training set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Plot feature distributions over different target values (house prices in our case).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="baseline---spark-mllib-linear-regressor">Baseline - Spark MLlib Linear Regressor<a href="#baseline---spark-mllib-linear-regressor" class="hash-link" aria-label="Direct link to Baseline - Spark MLlib Linear Regressor" title="Direct link to Baseline - Spark MLlib Linear Regressor">​</a></h2><p>First, we set a baseline performance by using Linear Regressor in Spark MLlib.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># By default, `maxIter` is 100. Other params you may want to change include: `regParam`, `elasticNetParam`, etc.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_model </span><span class="token operator">=</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr_predictions </span><span class="token operator">=</span><span class="token plain"> lr_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We evaluate the prediction result by using <code>synapse.ml.train.ComputeModelStatistics</code> which returns four metrics:</p><ul><li><a href="https://en.wikipedia.org/wiki/Mean_squared_error" target="_blank" rel="noopener noreferrer">MSE (Mean Squared Error)</a></li><li><a href="https://en.wikipedia.org/wiki/Root-mean-square_deviation" target="_blank" rel="noopener noreferrer">RMSE (Root Mean Squared Error)</a> = sqrt(MSE)</li><li><a href="https://en.wikipedia.org/wiki/Coefficient_of_determination" target="_blank" rel="noopener noreferrer">R Squared</a></li><li><a href="https://en.wikipedia.org/wiki/Mean_absolute_error" target="_blank" rel="noopener noreferrer">MAE (Mean Absolute Error)</a></li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib - Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpal-wabbit">Vowpal Wabbit<a href="#vowpal-wabbit" class="hash-link" aria-label="Direct link to Vowpal Wabbit" title="Direct link to Vowpal Wabbit">​</a></h2><p>Perform VW-style feature hashing. Many types (numbers, string, bool, map of string to (number, string)) are supported.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Use the same number of iterations as Spark MLlib&#x27;s Linear Regression (=100)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data_2 </span><span class="token operator">=</span><span class="token plain"> vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbm">LightGBM<a href="#lightgbm" class="hash-link" aria-label="Direct link to LightGBM" title="Direct link to LightGBM">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgr </span><span class="token operator">=</span><span class="token plain"> LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    objective</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;quantile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    learningRate</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numLeaves</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numIterations</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartitioned_data </span><span class="token operator">=</span><span class="token plain"> lr_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_model </span><span class="token operator">=</span><span class="token plain"> lgr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">repartitioned_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_predictions </span><span class="token operator">=</span><span class="token plain"> lg_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lr_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">results </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Following figure shows the actual-vs.-prediction graphs of the results:</p><img loading="lazy" width="1102" alt="lr-vw-lg" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/regression_comparison.png" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Spark MLlib Linear Regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBM&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lg_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">enumerate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">if</span><span class="token plain"> i </span><span class="token operator">==</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Train Regressor/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Train Regressor</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Isolation Forests</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#prepare-dataset" class="table-of-contents__link toc-highlight">Prepare Dataset</a></li><li><a href="#baseline---spark-mllib-linear-regressor" class="table-of-contents__link toc-highlight">Baseline - Spark MLlib Linear Regressor</a></li><li><a href="#vowpal-wabbit" class="table-of-contents__link toc-highlight">Vowpal Wabbit</a></li><li><a href="#lightgbm" class="table-of-contents__link toc-highlight">LightGBM</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html b/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html
index 18ac3e191d..0cdde4be19 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Data Balance Analysis on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="context">Context<a href="#context" class="hash-link" aria-label="Direct link to Context" title="Direct link to Context">​</a></h2><p>Data Balance Analysis is relevant for gaining an overall understanding of datasets, but it becomes essential when thinking about building AI systems in a responsible way, especially in terms of fairness.</p><p>AI systems can sometimes exhibit unwanted, unfair behaviors. These behaviors can cause fairness-related harms that affect various groups of people. They may amplify the marginalization of particular groups whose needs and contexts are often overlooked during AI development and deployment. Fairness-related harms can have varying severities, and the cumulative impact of even seemingly non-severe harms can be burdensome.</p><p>Fairness-related harms include:</p><ul><li><strong>Allocation harms</strong>: When an AI system extends or withholds opportunities or resources in ways that negatively impact people’s lives.</li><li><strong>Quality of service harms</strong>: When an AI system does not work as well for one group of people as it does for another.</li><li><strong>Stereotyping harms</strong>: When an AI system makes unfair generalizations about groups of people and reinforces negative stereotypes.</li><li><strong>Demeaning harms</strong>: When an AI system is actively derogatory or offensive.</li><li><strong>Over/underrepresentation harms</strong>: When an AI system over/underrepresents some groups of people or may even erase some groups entirely.</li></ul><p><strong>Note</strong>: <em>Because fairness in AI is fundamentally a sociotechnical challenge, it&#x27;s often impossible to fully “de-bias” an AI system. Instead, teams tasked with developing and deploying AI systems must work to identify, measure, and mitigate fairness-related harms as much as possible. Data Balance Analysis is a tool to help do so, in combination with others.</em></p><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.</p><p>In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces the costs of building ML through the early identification of data representation gaps. Before proceeding to train their models, data scientists can seek mitigation steps such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><ul><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Data Balance Analysis currently supports three transformers in the <code>synapse.ml.exploratory</code> namespace:</p><ul><li>FeatureBalanceMeasure - supervised (requires label column)</li><li>DistributionBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li><li>AggregateBalanceMeasure - unsupervised (doesn&#x27;t require label column)</li></ul><ol><li><p>Import all three transformers.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FeatureBalanceMeasure</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Load your dataset, define features of interest, and ensure that the label column is binary. The <code>FeatureBalanceMeasure</code> transformer currently only supports binary labels, but support for numerical labels will be added soon.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>FeatureBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features and call <code>setLabelCol</code> to set the binary label column. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>DistributionBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li><li><p>Create a <code>AggregateBalanceMeasure</code> transformer and call <code>setSensitiveCols</code> to set the list of sensitive features. Then, call the <code>transform</code> method with your dataset and visualize the resulting dataframe.</p><p>For example:</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">truncate</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></li></ol><p>Note: If you&#x27;re running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures by calling the built-in plotting features <code>display()</code>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="measure-explanations">Measure Explanations<a href="#measure-explanations" class="hash-link" aria-label="Direct link to Measure Explanations" title="Direct link to Measure Explanations">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="feature-balance-measures">Feature Balance Measures<a href="#feature-balance-measures" class="hash-link" aria-label="Direct link to Feature Balance Measures" title="Direct link to Feature Balance Measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at balanced probability.</p><p>In this context, we define a feature balance measure, called the parity, for label y. It is the difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><table><thead><tr><th>Association Metric</th><th>Family</th><th>Description</th><th>Interpretation/Formula</th><th>Reference</th></tr></thead><tbody><tr><td>Statistical Parity</td><td>Fairness</td><td>Proportion of each segment of a protected class (gender, for example) that should receive the positive outcome at equal rates.</td><td>Closer to zero means better parity. <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>D</mi><mi>P</mi><mo>=</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>M</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo><mo>−</mo><mi>P</mi><mo stretchy="false">(</mo><mi>Y</mi><mi mathvariant="normal">∣</mi><mi>A</mi><mo>=</mo><mi>F</mi><mi>e</mi><mi>m</mi><mi>a</mi><mi>l</mi><mi>e</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">DP = P(Y \vert A = Male) - P(Y \vert A = Female)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.68333em;vertical-align:0em"></span><span class="mord mathnormal" style="margin-right:0.02778em">D</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.10903em">M</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.22222em">Y</span><span class="mord">∣</span><span class="mord mathnormal">A</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel">=</span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">m</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">e</span><span class="mclose">)</span></span></span></span></span>.</td><td><a href="https://en.wikipedia.org/wiki/Fairness_%28machine_learning%29" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Pointwise Mutual Information (PMI), normalized PMI</td><td>Entropy</td><td>The PMI of a pair of feature values (ex: Gender=Male and Gender=Female) quantifies the discrepancy between the probability of their coincidence given their joint distribution and their individual distributions (assuming independence).</td><td>Range (normalized) <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mo>−</mo><mn>1</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[-1, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">−</span><span class="mord">1</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. -1 for no co-occurrences. 0 for co-occurrences at random. 1 for complete co-occurrences.</td><td><a href="https://en.wikipedia.org/wiki/Pointwise_mutual_information" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Sorensen-Dice Coefficient (SDC)</td><td>Intersection-over-Union</td><td>Used to gauge the similarity of two samples. Related to F1 score.</td><td>Equals twice the number of elements common to both sets divided by the sum of the number of elements in each set.</td><td><a href="https://en.wikipedia.org/wiki/S%C3%B8rensen%E2%80%93Dice_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Jaccard Index</td><td>Intersection-over-Union</td><td>Similar to SDC, gauges the similarity and diversity of sample sets.</td><td>Equals the size of the intersection divided by the size of the union of the sample sets.</td><td><a href="https://en.wikipedia.org/wiki/Jaccard_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Kendall Rank Correlation</td><td>Correlation and Statistical Tests</td><td>Used to measure the ordinal association between two measured quantities.</td><td>High when observations have a similar rank and low when observations have a dissimilar rank between the two variables.</td><td><a href="https://en.wikipedia.org/wiki/Kendall_rank_correlation_coefficient" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Log-Likelihood Ratio</td><td>Correlation and Statistical Tests</td><td>Calculates the degree to which data supports one variable versus another. Log of the likelihood ratio, which gives the probability of correctly predicting the label in ratio to probability of incorrectly predicting label.</td><td>If likelihoods are similar, it should be close to 0.</td><td><a href="https://en.wikipedia.org/wiki/Likelihood_function#Likelihood_ratio" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>t-test</td><td>Correlation and Statistical Tests</td><td>Used to compare the means of two groups (pairwise).</td><td>Value looked up in t-Distribution tell if statistically significant or not.</td><td><a href="https://en.wikipedia.org/wiki/Student&#x27;s_t-test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="distribution-balance-measures">Distribution Balance Measures<a href="#distribution-balance-measures" class="hash-link" aria-label="Direct link to Distribution Balance Measures" title="Direct link to Distribution Balance Measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (currently only uniform distribution is supported as a reference distribution). They are calculated per sensitive column and don&#x27;t depend on the label column.</p><p>For example, let&#x27;s assume we have a dataset with nine rows and a Gender column, and we observe that:</p><ul><li>&quot;Male&quot; appears four times</li><li>&quot;Female&quot; appears three times</li><li>&quot;Other&quot; appears twice</li></ul><p>Assuming the uniform distribution:</p><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>C</mi><mi>o</mi><mi>u</mi><mi>n</mi><mi>t</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>R</mi><mi>o</mi><mi>w</mi><mi>s</mi></mrow><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceCount \coloneqq \frac{numRows}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.07153em">C</span><span class="mord mathnormal">o</span><span class="mord mathnormal">u</span><span class="mord mathnormal">n</span><span class="mord mathnormal">t</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.04633em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.36033em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">o</span><span class="mord mathnormal" style="margin-right:0.02691em">w</span><span class="mord mathnormal">s</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><div class="math math-display"><span class="katex-display"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML" display="block"><semantics><mrow><mi>R</mi><mi>e</mi><mi>f</mi><mi>e</mi><mi>r</mi><mi>e</mi><mi>n</mi><mi>c</mi><mi>e</mi><mi>P</mi><mi>r</mi><mi>o</mi><mi>b</mi><mi>a</mi><mi>b</mi><mi>i</mi><mi>l</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo><mi mathvariant="normal">≔</mi></mo><mfrac><mn>1</mn><mrow><mi>n</mi><mi>u</mi><mi>m</mi><mi>F</mi><mi>e</mi><mi>a</mi><mi>t</mi><mi>u</mi><mi>r</mi><mi>e</mi><mi>V</mi><mi>a</mi><mi>l</mi><mi>u</mi><mi>e</mi><mi>s</mi></mrow></mfrac></mrow><annotation encoding="application/x-tex">ReferenceProbability \coloneqq \frac{1}{numFeatureValues}</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:0.8888799999999999em;vertical-align:-0.19444em"></span><span class="mord mathnormal" style="margin-right:0.00773em">R</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.10764em">f</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal">n</span><span class="mord mathnormal">c</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.13889em">P</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">o</span><span class="mord mathnormal">b</span><span class="mord mathnormal">a</span><span class="mord mathnormal">b</span><span class="mord mathnormal">i</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:2.00744em;vertical-align:-0.686em"></span><span class="mord"><span class="mopen nulldelimiter"></span><span class="mfrac"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:1.32144em"><span style="top:-2.314em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord mathnormal">n</span><span class="mord mathnormal">u</span><span class="mord mathnormal">m</span><span class="mord mathnormal" style="margin-right:0.13889em">F</span><span class="mord mathnormal">e</span><span class="mord mathnormal">a</span><span class="mord mathnormal">t</span><span class="mord mathnormal">u</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">e</span><span class="mord mathnormal" style="margin-right:0.22222em">V</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">u</span><span class="mord mathnormal">e</span><span class="mord mathnormal">s</span></span></span><span style="top:-3.23em"><span class="pstrut" style="height:3em"></span><span class="frac-line" style="border-bottom-width:0.04em"></span></span><span style="top:-3.677em"><span class="pstrut" style="height:3em"></span><span class="mord"><span class="mord">1</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.686em"><span></span></span></span></span></span><span class="mclose nulldelimiter"></span></span></span></span></span></span></div><table><thead><tr><th>Feature Value</th><th>Observed Count</th><th>Reference Count</th><th>Observed Probability</th><th>Reference Probabiliy</th></tr></thead><tbody><tr><td>Male</td><td>4</td><td>9/3 = 3</td><td>4/9 = 0.44</td><td>3/9 = 0.33</td></tr><tr><td>Female</td><td>3</td><td>9/3 = 3</td><td>3/9 = 0.33</td><td>3/9 = 0.33</td></tr><tr><td>Other</td><td>2</td><td>9/3 = 3</td><td>2/9 = 0.22</td><td>3/9 = 0.33</td></tr></tbody></table><p>We can use distance measures to find out how far our observed and reference distributions of these feature values are. Some of these distance measures include:</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>KL Divergence</td><td>Measure of how one probability distribution is different from a second, reference probability distribution. Measure of the information gained when one revises one&#x27;s beliefs from the prior probability distribution Q to the posterior probability distribution P. In other words, it is the amount of information lost when Q is used to approximate P.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Kullback%E2%80%93Leibler_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>JS Distance</td><td>Measuring the similarity between two probability distributions. Symmetrized and smoothed version of the Kullback–Leibler (KL) divergence. Square root of JS Divergence.</td><td>Range <!-- -->[0, 1]<!-- -->. 0 means perfectly same to balanced distribution.</td><td><a href="https://en.wikipedia.org/wiki/Jensen%E2%80%93Shannon_divergence" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Wasserstein Distance</td><td>This distance is also known as the earth mover’s distance, since it can be seen as the minimum amount of “work” required to transform u into v, where “work” is measured as the amount of distribution weight that must be moved multiplied by the distance it has to be moved.</td><td>Non-negative. 0 means P = Q.</td><td><a href="https://en.wikipedia.org/wiki/Wasserstein_metric" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Infinity Norm Distance</td><td>Distance between two vectors is the greatest of their differences along any coordinate dimension. Also called Chebyshev distance or chessboard distance.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Chebyshev_distance" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Total Variation Distance</td><td>It is equal to half the L1 (Manhattan) distance between the two distributions. Take the difference between the two proportions in each category, add up the absolute values of all the differences, and then divide the sum by 2.</td><td>Non-negative. 0 means same distribution.</td><td><a href="https://en.wikipedia.org/wiki/Total_variation_distance_of_probability_measures" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Chi-Squared Test</td><td>The chi-square test tests the null hypothesis that the categorical data has the given frequencies given expected frequencies in each category.</td><td>p-value gives evidence against null-hypothesis that difference in observed and expected frequencies is by random chance.</td><td><a href="https://en.wikipedia.org/wiki/Chi-squared_test" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="aggregate-balance-measures">Aggregate Balance Measures<a href="#aggregate-balance-measures" class="hash-link" aria-label="Direct link to Aggregate Balance Measures" title="Direct link to Aggregate Balance Measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They&#x27;re calculated on the set of all sensitive columns and don&#x27;t depend on the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are specified as sensitive features, it then tries to quantify imbalance across all combinations of the two specified features - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><table><thead><tr><th>Measure</th><th>Description</th><th>Interpretation</th><th>Reference</th></tr></thead><tbody><tr><td>Atkinson Index</td><td>It presents the percentage of total income that a given society would have to forego in order to have more equal shares of income between its citizens. This measure depends on the degree of societal aversion to inequality (a theoretical parameter decided by the researcher). A higher value entails greater social utility or willingness by individuals to accept smaller incomes in exchange for a more equal distribution. An important feature of the Atkinson index is that it can be decomposed into within-group and between-group inequality.</td><td>Range <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><mn>0</mn><mo separator="true">,</mo><mn>1</mn><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[0, 1]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord">0</span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord">1</span><span class="mclose">]</span></span></span></span></span>. 0 if perfect equality. 1 means maximum inequality. In our case, it is the proportion of records for a sensitive columns’ combination.</td><td><a href="https://en.wikipedia.org/wiki/Atkinson_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil T Index</td><td>GE(1) = Theil&#x27;s T and is more sensitive to differences at the top of the distribution. The Theil index is a statistic used to measure economic inequality. The Theil index measures an entropic &quot;distance&quot; the population is away from the &quot;ideal&quot; egalitarian state of everyone having the same income.</td><td>If everyone has the same income, then T_T equals 0. If one person has all the income, then T_T gives the result <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>l</mi><mi>n</mi><mo stretchy="false">(</mo><mi>N</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">ln(N)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal" style="margin-right:0.01968em">l</span><span class="mord mathnormal">n</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.10903em">N</span><span class="mclose">)</span></span></span></span></span>. 0 means equal income and larger values mean higher level of disproportion.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr><tr><td>Theil L Index</td><td>GE(0) = Theil&#x27;s L and is more sensitive to differences at the lower end of the distribution. Logarithm of (mean income)/(income i), over all the incomes included in the summation. It is also referred to as the mean log deviation measure. Because a transfer from a larger income to a smaller one will change the smaller income&#x27;s ratio more than it changes the larger income&#x27;s ratio, the transfer-principle is satisfied by this index.</td><td>Same interpretation as Theil T Index.</td><td><a href="https://en.wikipedia.org/wiki/Theil_index" target="_blank" rel="noopener noreferrer">Link</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="mitigation">Mitigation<a href="#mitigation" class="hash-link" aria-label="Direct link to Mitigation" title="Direct link to Mitigation">​</a></h2><p>It will not be a stretch to say that every real-world dataset has caveats, biases, and imbalances. Data collection is costly. Data Imbalance mitigation or de-biasing data is an area of research. There are many techniques available at various stages of ML lifecycle: during pre-processing, in-processing, and post processing. Here we outline a couple of pre-processing techniques -</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="resampling">Resampling<a href="#resampling" class="hash-link" aria-label="Direct link to Resampling" title="Direct link to Resampling">​</a></h3><p>Resampling involves under-sampling from majority class and over-sampling from minority class. A naïve way to over-sample would be to duplicate records. Similarly, to under-sample one could remove records at random.</p><ul><li><p>Caveats:</p><ol><li>Under-sampling may remove valuable information.</li><li>Over-sampling may cause overfitting and poor generalization on test set.</li></ol></li></ul><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SamplingBar.png" alt="Bar chart undersampling and oversampling" class="img_ev3q"></p><p>There are smarter techniques to under-sample and over-sample in literature and implemented in Python’s <a href="https://imbalanced-learn.org/stable/" target="_blank" rel="noopener noreferrer">imbalanced-learn</a> package.</p><p>For example, we can cluster the records of the majority class, and do the under-sampling by removing records from each cluster, thus seeking to preserve information.</p><p>One technique of under-sampling is use of Tomek Links. Tomek links are pairs of instances that are very close but of opposite classes. Removing the instances of the majority class of each pair increases the space between the two classes, facilitating the classification process. A similar way to under-sample majority class is using Near-Miss. It first calculates the distance between all the points in the larger class with the points in the smaller class. When two points belonging to different classes are very close to each other in the distribution, this algorithm eliminates the datapoint of the larger class thereby trying to balance the distribution.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_TomekLinks.png" alt="Tomek Links" class="img_ev3q"></p><p>In over-sampling, instead of creating exact copies of the minority class records, we can introduce small variations into those copies, creating more diverse synthetic samples. This technique is called SMOTE (Synthetic Minority Oversampling Technique). It randomly picks a point from the minority class and computes the k-nearest neighbors for this point. The synthetic points are added between the chosen point and its neighbors.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_SyntheticSamples.png" alt="Synthetic Samples" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="reweighting">Reweighting<a href="#reweighting" class="hash-link" aria-label="Direct link to Reweighting" title="Direct link to Reweighting">​</a></h3><p>There is an expected and observed value in each table cell. The weight is the value of expected / observed. Reweighting is easy to extend to multiple features with more than two groups. The weights are then incorporated in loss function of model training.</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_Reweight.png" alt="Reweighting" class="img_ev3q"></p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Explanation Dashboard</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#context" class="table-of-contents__link toc-highlight">Context</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#measure-explanations" class="table-of-contents__link toc-highlight">Measure Explanations</a><ul><li><a href="#feature-balance-measures" class="table-of-contents__link toc-highlight">Feature Balance Measures</a></li><li><a href="#distribution-balance-measures" class="table-of-contents__link toc-highlight">Distribution Balance Measures</a></li><li><a href="#aggregate-balance-measures" class="table-of-contents__link toc-highlight">Aggregate Balance Measures</a></li></ul></li><li><a href="#mitigation" class="table-of-contents__link toc-highlight">Mitigation</a><ul><li><a href="#resampling" class="table-of-contents__link toc-highlight">Resampling</a></li><li><a href="#reweighting" class="table-of-contents__link toc-highlight">Reweighting</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html b/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html
index 90221c5743..a880605fd5 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Explanation Dashboard/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Explanation Dashboard/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Explanation Dashboard/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Explanation Dashboard</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---explanation-dashboard">Interpretability - Explanation Dashboard<a href="#interpretability---explanation-dashboard" class="hash-link" aria-label="Direct link to Interpretability - Explanation Dashboard" title="Direct link to Interpretability - Explanation Dashboard">​</a></h2><p>In this example, similar to the &quot;Interpretability - Tabular SHAP explainer&quot; notebook, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset and then visualize the explanation in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets</a>.</p><p>First we import the packages and define some UDFs we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token operator">%</span><span class="token plain">pip install raiwidgets itsdangerous</span><span class="token operator">==</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token plain"> interpret</span><span class="token operator">-</span><span class="token plain">community</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">terminal</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">interactiveshell </span><span class="token keyword">import</span><span class="token plain"> TerminalInteractiveShell</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a simple binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we are trying to explain. In this case, we are trying to explain the &quot;probability&quot; output which is a vector of length 2, and we are only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We can visualize the explanation in the <a href="https://github.com/interpretml/interpret-community" target="_blank" rel="noopener noreferrer">interpret-community format</a> in the ExplanationDashboard from <a href="https://github.com/microsoft/responsible-ai-widgets/" target="_blank" rel="noopener noreferrer">https://github.com/microsoft/responsible-ai-widgets/</a></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">local_importance_values </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_data </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">true_y </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">list_local_importance_values </span><span class="token operator">=</span><span class="token plain"> local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">converted_importance_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bias </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> classarray </span><span class="token keyword">in</span><span class="token plain"> list_local_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rowarray </span><span class="token keyword">in</span><span class="token plain"> classarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_list </span><span class="token operator">=</span><span class="token plain"> rowarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        bias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token comment" style="color:rgb(136, 132, 111)"># remove the bias from local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">del</span><span class="token plain"> converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">converted_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>When running Synapse Analytics, please follow instructions here <a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/spark/apache-spark-azure-portal-add-libraries" target="_blank" rel="noopener noreferrer">Package management - Azure Synapse Analytics | Microsoft Docs</a> to install <a href="https://pypi.org/project/raiwidgets/" target="_blank" rel="noopener noreferrer">&quot;raiwidgets&quot;</a> and <a href="https://pypi.org/project/interpret-community/" target="_blank" rel="noopener noreferrer">&quot;interpret-community&quot;</a> packages.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> interpret_community</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">adapter </span><span class="token keyword">import</span><span class="token plain"> ExplanationAdapter</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> ExplanationAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classification</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation </span><span class="token operator">=</span><span class="token plain"> adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">create_global</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    converted_importance_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> expected_values</span><span class="token operator">=</span><span class="token plain">bias</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the global importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">global_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the local importance values</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">local_importance_values</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">object</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">__init__</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">predict_proba</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">self</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sparkdata </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sparkdata</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        proba_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> vector </span><span class="token keyword">in</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> proba_list</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># view the explanation in the ExplanationDashboard</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> raiwidgets </span><span class="token keyword">import</span><span class="token plain"> ExplanationDashboard</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ExplanationDashboard</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    global_explanation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> wrapper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dataset</span><span class="token operator">=</span><span class="token plain">eval_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> true_y</span><span class="token operator">=</span><span class="token plain">true_y</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/rai-dashboard.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---explanation-dashboard" class="table-of-contents__link toc-highlight">Interpretability - Explanation Dashboard</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Image Explainers/index.html b/docs/next/Explore Algorithms/Responsible AI/Image Explainers/index.html
index 8f244dfb2b..14ed5df367 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Image Explainers/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Image Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Image Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Image Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Image Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Image Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Image Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Image Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Image Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---image-explainers">Interpretability - Image Explainers<a href="#interpretability---image-explainers" class="hash-link" aria-label="Direct link to Interpretability - Image Explainers" title="Direct link to Interpretability - Image Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain the ResNet50 model&#x27;s multi-class output of an image.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> ImageTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> PIL </span><span class="token keyword">import</span><span class="token plain"> Image</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_slice </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">indices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">arg_top_k </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">lambda</span><span class="token plain"> vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">argsort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain">k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tolist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ArrayType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">return</span><span class="token plain"> barr</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">_ </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bgr_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">reshape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">R</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> G</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> B</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> rgb_image_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sp_clusters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> sp_clusters</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    green_value </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> green_threshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fromarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_rgb_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> v </span><span class="token operator">&gt;</span><span class="token plain"> green_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Create a dataframe for a testing image, and use the ResNet50 ONNX model to infer the image.</p><p>The result shows 39.6% probability of &quot;violin&quot; (889), and 38.4% probability of &quot;upright piano&quot; (881).</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Rotate the image array from BGR into RGB channels for visualization later.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">row </span><span class="token operator">=</span><span class="token plain"> image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.height&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.nChannels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image.data&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token builtin" style="color:rgb(174, 129, 255)">locals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rgb_image_array </span><span class="token operator">=</span><span class="token plain"> rotate_color_channel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> nChannels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Download the ONNX model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelPayload </span><span class="token operator">=</span><span class="token plain"> downloadBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/publicwasb/ONNXModels/resnet50-v2-7.onnx&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        color_scale_factor</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelPayload</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;resnetv24_dense0_fwd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSoftMaxDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMiniBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onnx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predicted </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> arg_top_k</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_slice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_row </span><span class="token operator">=</span><span class="token plain"> lime_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the LIME weights for &quot;violin&quot; output and &quot;upright piano&quot; output.</p><p>Green areas are superpixels with LIME weights above 95 percentile.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lime_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-lime-20210811.png" class="img_ev3q"><p>Then we use the Kernel SHAP image explainer to explain the model&#x27;s top 2 classes&#x27; probabilities.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_result </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predicted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_row </span><span class="token operator">=</span><span class="token plain"> shap_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">head</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We plot the SHAP values for &quot;piano&quot; output and &quot;cell&quot; output.</p><p>Green areas are superpixels with SHAP values above 95 percentile.</p><blockquote><p>Notice that we drop the base value from the SHAP output before rendering the superpixels. The base value is the model output for the background (all black) image.</p></blockquote><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_violin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rgb_image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">shap_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps_piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">95</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/image-shap-20210811.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Text Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">PDP and ICE Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---image-explainers" class="table-of-contents__link toc-highlight">Interpretability - Image Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html b/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html
index fca27d78df..cc0a5366e3 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Interpreting Model Predictions</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Model Interpretation on Spark</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretable-machine-learning">Interpretable Machine Learning<a href="#interpretable-machine-learning" class="hash-link" aria-label="Direct link to Interpretable Machine Learning" title="Direct link to Interpretable Machine Learning">​</a></h2><p>Interpretable Machine Learning helps developers, data scientists and business stakeholders in the organization gain a comprehensive understanding of their machine learning models. It can also be used to debug models, explain predictions and enable auditing to meet compliance with regulatory requirements.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="why-run-model-interpretation-on-spark">Why run model interpretation on Spark<a href="#why-run-model-interpretation-on-spark" class="hash-link" aria-label="Direct link to Why run model interpretation on Spark" title="Direct link to Why run model interpretation on Spark">​</a></h2><p>Model-agnostic interpretation methods can be computationally expensive due to the multiple evaluations needed to compute the explanations. Model interpretation on Spark enables users to interpret a black-box model at massive scales with the Apache Spark™ distributed computing ecosystem. Various components support local interpretation for tabular, vector, image and text classification models, with two popular model-agnostic interpretation methods: <a href="https://arxiv.org/abs/1602.04938" target="_blank" rel="noopener noreferrer">LIME</a> and <a href="https://arxiv.org/abs/1705.07874" target="_blank" rel="noopener noreferrer">Kernel SHAP</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="usage">Usage<a href="#usage" class="hash-link" aria-label="Direct link to Usage" title="Direct link to Usage">​</a></h2><p>Both LIME and Kernel SHAP are local interpretation methods. Local interpretation explains why does the model predict certain outcome for a given observation.</p><p>Both explainers extends from <code>org.apache.spark.ml.Transformer</code>. After setting up the explainer parameters, simply call the <code>transform</code> function on a <code>DataFrame</code> of observations to interpret the model behavior on these observations.</p><p>To see examples of model interpretability on Spark in action, take a look at these sample notebooks:</p><ul><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li></ul><table><thead><tr><th></th><th>Tabular models</th><th>Vector models</th><th>Image models</th><th>Text models</th></tr></thead><tbody><tr><td>LIME explainers</td><td><a href="#tabularlime">TabularLIME</a></td><td><a href="#vectorlime">VectorLIME</a></td><td><a href="#imagelime">ImageLIME</a></td><td><a href="#textlime">TextLIME</a></td></tr><tr><td>Kernel SHAP explainers</td><td><a href="#tabularshap">TabularSHAP</a></td><td><a href="#vectorshap">VectorSHAP</a></td><td><a href="#imageshap">ImageSHAP</a></td><td><a href="#textshap">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-local-explainer-params">Common local explainer params<a href="#common-local-explainer-params" class="hash-link" aria-label="Direct link to Common local explainer params" title="Direct link to Common local explainer params">​</a></h3><p>All local explainers support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>targetCol</td><td><code>String</code></td><td>&quot;probability&quot;</td><td>The column name of the prediction target to explain (i.e. the response variable).  This is usually set to &quot;prediction&quot; for regression models and &quot;probability&quot; for probabilistic classification models.</td></tr><tr><td>targetClasses</td><td><code>Array[Int]</code></td><td>empty array</td><td>The indices of the classes for multinomial classification models.</td></tr><tr><td>targetClassesCol</td><td><code>String</code></td><td></td><td>The name of the column that specifies the indices of the classes for multinomial classification models.</td></tr><tr><td>outputCol</td><td><code>String</code></td><td></td><td>The name of the output column for interpretation results.</td></tr><tr><td>model</td><td><code>Transformer</code></td><td></td><td>The model to be explained.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-lime-explainer-params">Common LIME explainer params<a href="#common-lime-explainer-params" class="hash-link" aria-label="Direct link to Common LIME explainer params" title="Direct link to Common LIME explainer params">​</a></h3><p>All LIME based explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#vectorlime">VectorLIME</a>, <a href="#imagelime">ImageLIME</a>, <a href="#textlime">TextLIME</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>regularization</td><td><code>Double</code></td><td>0</td><td>Regularization param for the underlying lasso regression.</td></tr><tr><td>kernelWidth</td><td><code>Double</code></td><td>sqrt(number of features) * 0.75</td><td>Kernel width for the exponential kernel.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>1000</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="common-shap-explainer-params">Common SHAP explainer params<a href="#common-shap-explainer-params" class="hash-link" aria-label="Direct link to Common SHAP explainer params" title="Direct link to Common SHAP explainer params">​</a></h3><p>All Kernel SHAP based explainers (<a href="#tabularshap">TabularSHAP</a>, <a href="#vectorshap">VectorSHAP</a>, <a href="#imageshap">ImageSHAP</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>infWeight</td><td><code>Double</code></td><td>1E8</td><td>The double value to represent infinite weight.</td></tr><tr><td>numSamples</td><td><code>Int</code></td><td>2 * (number of features) + 2048</td><td>Number of samples to generate.</td></tr><tr><td>metricsCol</td><td><code>String</code></td><td>&quot;r2&quot;</td><td>Column name for fitting metrics.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabular-model-explainer-params">Tabular model explainer params<a href="#tabular-model-explainer-params" class="hash-link" aria-label="Direct link to Tabular model explainer params" title="Direct link to Tabular model explainer params">​</a></h3><p>All tabular model explainers (<a href="#tabularlime">TabularLIME</a>, <a href="#tabularshap">TabularSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCols</td><td><code>Array[String]</code></td><td></td><td>The names of input columns to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain all the input columns needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vector-model-explainer-params">Vector model explainer params<a href="#vector-model-explainer-params" class="hash-link" aria-label="Direct link to Vector model explainer params" title="Direct link to Vector model explainer params">​</a></h3><p>All vector model explainers (<a href="#vectorlime">VectorLIME</a>, <a href="#vectorshap">VectorSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input vector column to the black-box model.</td></tr><tr><td>backgroundData</td><td><code>DataFrame</code></td><td></td><td>A dataframe containing background data. It must contain the input vector column needed by the black-box model.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="image-model-explainer-params">Image model explainer params<a href="#image-model-explainer-params" class="hash-link" aria-label="Direct link to Image model explainer params" title="Direct link to Image model explainer params">​</a></h3><p>All image model explainers (<a href="#imagelime">ImageLIME</a>, <a href="#imageshap">ImageSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input image column to the black-box model.</td></tr><tr><td>cellSize</td><td><code>Double</code></td><td>16</td><td>Number that controls the size of the super-pixels.</td></tr><tr><td>modifier</td><td><code>Double</code></td><td>130</td><td>Controls the trade-off spatial and color distance of super-pixels.</td></tr><tr><td>superpixelCol</td><td><code>String</code></td><td>&quot;superpixels&quot;</td><td>The column holding the super-pixel decompositions.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="text-model-explainer-params">Text model explainer params<a href="#text-model-explainer-params" class="hash-link" aria-label="Direct link to Text model explainer params" title="Direct link to Text model explainer params">​</a></h3><p>All text model explainers (<a href="#textlime">TextLIME</a>, <a href="#textshap">TextSHAP</a>) support the following params:</p><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>inputCol</td><td><code>String</code></td><td></td><td>The names of input text column to the black-box model.</td></tr><tr><td>tokensCol</td><td><code>String</code></td><td>&quot;tokens&quot;</td><td>The column holding the text tokens.</td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime"><code>TabularLIME</code><a href="#tabularlime" class="hash-link" aria-label="Direct link to tabularlime" title="Direct link to tabularlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>categoricalFeatures</td><td><code>Array[String]</code></td><td>empty array</td><td>The name of columns that should be treated as categorical variables.</td></tr></tbody></table><blockquote><p>For categorical features, <code>TabularLIME</code> creates new samples by drawing samples based on the value distribution from the background dataset. For numerical features, it creates new samples by drawing from a normal distribution with mean taken from the target value to be explained, and standard deviation taken from the background dataset.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap"><code>TabularSHAP</code><a href="#tabularshap" class="hash-link" aria-label="Direct link to tabularshap" title="Direct link to tabularshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime"><code>VectorLIME</code><a href="#vectorlime" class="hash-link" aria-label="Direct link to vectorlime" title="Direct link to vectorlime">​</a></h3><p>No additional params are supported.</p><blockquote><p><code>VectorLIME</code> assumes all features are numerical, and categorical features are not supported in <code>VectorLIME</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap"><code>VectorSHAP</code><a href="#vectorshap" class="hash-link" aria-label="Direct link to vectorshap" title="Direct link to vectorshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime"><code>ImageLIME</code><a href="#imagelime" class="hash-link" aria-label="Direct link to imagelime" title="Direct link to imagelime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of super-pixels to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>ImageLIME</code> creates new samples by randomly turning super-pixels on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap"><code>ImageSHAP</code><a href="#imageshap" class="hash-link" aria-label="Direct link to imageshap" title="Direct link to imageshap">​</a></h3><p>No additional params are supported.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime"><code>TextLIME</code><a href="#textlime" class="hash-link" aria-label="Direct link to textlime" title="Direct link to textlime">​</a></h3><table><thead><tr><th>Param</th><th>Type</th><th>Default</th><th>Description</th></tr></thead><tbody><tr><td>samplingFraction</td><td><code>Double</code></td><td>0.7</td><td>The fraction of word tokens to keep on during sampling.</td></tr></tbody></table><blockquote><p><code>TextLIME</code> creates new samples by randomly turning word tokens on or off with probability of keeping on set to <code>SamplingFraction</code>.</p></blockquote><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap"><code>TextSHAP</code><a href="#textshap" class="hash-link" aria-label="Direct link to textshap" title="Direct link to textshap">​</a></h3><p>No additional params are supported.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="result-interpretation">Result interpretation<a href="#result-interpretation" class="hash-link" aria-label="Direct link to Result interpretation" title="Direct link to Result interpretation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lime-explainers">LIME explainers<a href="#lime-explainers" class="hash-link" aria-label="Direct link to LIME explainers" title="Direct link to LIME explainers">​</a></h3><p>LIME explainers return an array of vectors, and each vector maps to a class being explained. Each component of the vector is the coefficient for the corresponding feature, super-pixel, or word token from the local surrogate model.</p><ul><li>For categorical variables, super-pixels, or word tokens, the coefficient shows the average change in model outcome if this feature is unknown to the model, if the super-pixel is replaced with background color (black), or if the word token is replaced with empty string.</li><li>For numeric variables, the coefficient shows the change in model outcome if the feature value is incremented by 1 unit.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="shap-explainers">SHAP explainers<a href="#shap-explainers" class="hash-link" aria-label="Direct link to SHAP explainers" title="Direct link to SHAP explainers">​</a></h3><p>SHAP explainers return an array of vectors, and each vector maps to a class being explained. Each vector starts with the <a href="#base-value">base value</a>, and each following component of the vector is the Shapley value for each feature, super-pixel, or token.</p><p>The base value and Shapley values are additive, and they should add up to the model output for the target observation.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="base-value">Base value<a href="#base-value" class="hash-link" aria-label="Direct link to Base value" title="Direct link to Base value">​</a></h4><ul><li>For tabular and vector models, the base value represents the mean outcome of the model for the background dataset.</li><li>For image models, the base value represents the model outcome for a background (all black) image.</li><li>For text models, the base value represents the model outcome for an empty string.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Quickstart - Transfer Learn for Image Classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Transfer Learn for Image Classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Tabular Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretable-machine-learning" class="table-of-contents__link toc-highlight">Interpretable Machine Learning</a></li><li><a href="#why-run-model-interpretation-on-spark" class="table-of-contents__link toc-highlight">Why run model interpretation on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a><ul><li><a href="#common-local-explainer-params" class="table-of-contents__link toc-highlight">Common local explainer params</a></li><li><a href="#common-lime-explainer-params" class="table-of-contents__link toc-highlight">Common LIME explainer params</a></li><li><a href="#common-shap-explainer-params" class="table-of-contents__link toc-highlight">Common SHAP explainer params</a></li><li><a href="#tabular-model-explainer-params" class="table-of-contents__link toc-highlight">Tabular model explainer params</a></li><li><a href="#vector-model-explainer-params" class="table-of-contents__link toc-highlight">Vector model explainer params</a></li><li><a href="#image-model-explainer-params" class="table-of-contents__link toc-highlight">Image model explainer params</a></li><li><a href="#text-model-explainer-params" class="table-of-contents__link toc-highlight">Text model explainer params</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight"><code>TabularLIME</code></a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight"><code>TabularSHAP</code></a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight"><code>VectorLIME</code></a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight"><code>VectorSHAP</code></a></li><li><a href="#imagelime" class="table-of-contents__link toc-highlight"><code>ImageLIME</code></a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight"><code>ImageSHAP</code></a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight"><code>TextLIME</code></a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight"><code>TextSHAP</code></a></li></ul></li><li><a href="#result-interpretation" class="table-of-contents__link toc-highlight">Result interpretation</a><ul><li><a href="#lime-explainers" class="table-of-contents__link toc-highlight">LIME explainers</a></li><li><a href="#shap-explainers" class="table-of-contents__link toc-highlight">SHAP explainers</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html b/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html
index b07269c4a1..decabdd6ad 100644
--- a/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">PDP and ICE Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-pdp-and-individual-conditional-expectation-ice-plots">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots<a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="hash-link" aria-label="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots" title="Direct link to Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots">​</a></h2><p>Partial Dependence Plot (PDP) and Individual Condition Expectation (ICE) are interpretation methods which describe the average behavior of a classification or regression model. They are particularly useful when the model developer wants to understand generally how the model depends on individual feature values, overall model behavior and do debugging.</p><p>To practice responsible AI, it is crucial to understand which features drive your model&#x27;s predictions. This knowledge can facilitate the creation of Transparency Notes, facilitate auditing and compliance, help satisfy regulatory requirements, and improve both transparency and accountability.</p><p>The goal of this notebook is to show how these methods work for a pretrained model.</p><p>In this example, we train a classification model with the Adult Census Income dataset. Then we treat the model as an opaque-box model and calculate the PDP and ICE plots for some selected categorical and numeric features. </p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><p>We will train a classification model to predict &gt;= 50K or &lt; 50K based on our features.</p><hr><p>Python dependencies:</p><p>matplotlib==3.2.2</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluation </span><span class="token keyword">import</span><span class="token plain"> BinaryClassificationEvaluator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ICETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="read-and-prepare-the-dataset">Read and prepare the dataset<a href="#read-and-prepare-the-dataset" class="hash-link" aria-label="Direct link to Read and prepare the dataset" title="Direct link to Read and prepare the dataset">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fit-the-model-and-view-the-predictions">Fit the model and view the predictions<a href="#fit-the-model-and-view-the-predictions" class="hash-link" aria-label="Direct link to Fit the model and view the predictions" title="Direct link to Fit the model and view the predictions">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">string_indexer_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringOrderType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">string_indexer_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">one_hot_encoder_outputs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token plain">one_hot_encoder_outputs </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxDepth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Check that model makes sense and has reasonable output. For this, we will check the model performance by calculating the ROC-AUC score.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc </span><span class="token operator">=</span><span class="token plain"> BinaryClassificationEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rawPredictionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">eval_auc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">evaluate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="partial-dependence-plots">Partial Dependence Plots<a href="#partial-dependence-plots" class="hash-link" aria-label="Direct link to Partial Dependence Plots" title="Direct link to Partial Dependence Plots">​</a></h2><p>Partial dependence plots (PDP) show the dependence between the target response and a set of input features of interest, marginalizing over the values of all other input features. It can show whether the relationship between the target response and the input feature is linear, smooth, monotonic, or more complex. This is relevant when you want to have an overall understanding of model behavior. E.g. Identifying specific age group has a favorable predictions vs other age groups.</p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#partial-dependence-plots" target="_blank" rel="noopener noreferrer">scikit-learn page on partial dependence plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp">Set up the transformer for PDP<a href="#set-up-the-transformer-for-pdp" class="hash-link" aria-label="Direct link to Set up the transformer for PDP" title="Direct link to Set up the transformer for PDP">​</a></h3><p>To plot PDP we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>average</code> and then call the <code>transform</code> function. </p><p>For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names.</p><p>Categorical and numeric features can be passed as a list of names. But we can specify parameters for the features by passing a list of dicts where each dict represents one feature. </p><p>For the numeric features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;capital-gain&quot;, &quot;numSplits&quot;: 20, &quot;rangeMin&quot;: 0.0, &quot;rangeMax&quot;: 10000.0, &quot;outputColName&quot;: &quot;capital-gain_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numSplits</code> - the number of splits for the value range for the numeric feature, <code>rangeMin</code> - specifies the min value of the range for the numeric feature, <code>rangeMax</code> - specifies the max value of the range for the numeric feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><p>For the categorical features a dictionary can look like this:</p><p>{&quot;name&quot;: &quot;marital-status&quot;, &quot;numTopValues&quot;: 10, &quot;outputColName&quot;: &quot;marital-status_dependance&quot;}</p><p>Where the required key-value pair is <code>name</code> -  the name of the numeric feature. Next key-values pairs are optional: <code>numTopValues</code> - the max number of top-occurring values to be included in the categorical feature, <code>outputColName</code> - the name for output column with explanations for the feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>PDP transformer returns a dataframe of 1 row * {number features to explain} columns. Each column contains a map between the feature&#x27;s values and the model&#x27;s average dependence for that feature value.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp </span><span class="token operator">=</span><span class="token plain"> pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization">Visualization<a href="#visualization" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">map_keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    keys </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    key_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    final_cols </span><span class="token operator">=</span><span class="token plain"> key_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pandas_df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">final_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> pandas_df</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">keys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-age">Example 1: &quot;age&quot;<a href="#example-1-age" class="hash-link" aria-label="Direct link to Example 1: &quot;age&quot;" title="Direct link to Example 1: &quot;age&quot;">​</a></h4><p>We can observe non-linear dependency. The model predicts that income rapidly grows from 24-46 y.o. age, after 46 y.o. model predictions slightly drops and from 68 y.o. remains stable.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age.png" alt="pdp_age" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-marital-status">Example 2: &quot;marital-status&quot;<a href="#example-2-marital-status" class="hash-link" aria-label="Direct link to Example 2: &quot;marital-status&quot;" title="Direct link to Example 2: &quot;marital-status&quot;">​</a></h4><p>The model seems to treat &quot;married-cv-spouse&quot; as one category and tend to give a higher average prediction, and all others as a second category with the lower average prediction.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_occupation </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_occupation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_marital-status.png" alt="pdp_marital-status" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-3-capital-gain">Example 3: &quot;capital-gain&quot;<a href="#example-3-capital-gain" class="hash-link" aria-label="Direct link to Example 3: &quot;capital-gain&quot;" title="Direct link to Example 3: &quot;capital-gain&quot;">​</a></h4><p>In the first graph, we run PDP with default parameters. We can see that this representation is not super useful because it is not granular enough. By default the range of numeric features are calculated dynamically from the data.</p><p>In the second graph, we set rangeMin = 0 and rangeMax = 10000 to visualize more granular interpretations for the feature of interest. Now we can see more clearly how the model made decisions in a smaller region.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-first.png" alt="pdp_capital-gain-first" class="img_ev3q"></p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;average&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numSplits&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rangeMax&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_cap_gain </span><span class="token operator">=</span><span class="token plain"> pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_education_num_gain </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_pdp_cap_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_dependence_for_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_education_num_gain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_capital-gain-second.png" alt="pdp_capital-gain-second" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions">Conclusions<a href="#conclusions" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>PDP can be used to show how features influence model predictions on average and help modeler catch unexpected behavior from the model.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="individual-conditional-expectation">Individual Conditional Expectation<a href="#individual-conditional-expectation" class="hash-link" aria-label="Direct link to Individual Conditional Expectation" title="Direct link to Individual Conditional Expectation">​</a></h2><p>ICE plots display one line per instance that shows how the instance’s prediction changes when a feature values change. Each line represents the predictions for one instance if we vary the feature of interest. This is relevant when you want to observe model prediction for instances individually in more details. </p><p>If you want to learn more please check out the <a href="https://scikit-learn.org/stable/modules/partial_dependence.html#individual-conditional-expectation-ice-plot" target="_blank" rel="noopener noreferrer">scikit-learn page on ICE plots</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-ice">Set up the transformer for ICE<a href="#set-up-the-transformer-for-ice" class="hash-link" aria-label="Direct link to Set up the transformer for ICE" title="Direct link to Set up the transformer for ICE">​</a></h3><p>To plot ICE we need to set up the instance of <code>ICETransformer</code> first and set the <code>kind</code> parameter to <code>individual</code> and then call the <code>transform</code> function. For the setup we need to pass the pretrained model, specify the target column (&quot;probability&quot; in our case), and pass categorical and numeric feature names. For better visualization we set the number of samples to 50.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">ice </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output </span><span class="token operator">=</span><span class="token plain"> ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-1">Visualization<a href="#visualization-1" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> math </span><span class="token keyword">import</span><span class="token plain"> pi</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> collections </span><span class="token keyword">import</span><span class="token plain"> defaultdict</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_numeric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">n </span><span class="token operator">/</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> pi </span><span class="token keyword">for</span><span class="token plain"> n </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    angles </span><span class="token operator">+=</span><span class="token plain"> angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">111</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> polar</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">=</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        values </span><span class="token operator">+=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fill</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">angles</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;teal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col_int</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values </span><span class="token operator">=</span><span class="token plain"> defaultdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names_ice </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    num_instances </span><span class="token operator">=</span><span class="token plain"> df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    instances_y </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">num_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dict_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dict_values_pdp </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    col_names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> col_name </span><span class="token keyword">in</span><span class="token plain"> col_names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        marklist </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">sorted</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            dict_values_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">items</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> key</span><span class="token operator">=</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">int</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">if</span><span class="token plain"> col_int </span><span class="token keyword">else</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        sortdict </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">dict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">marklist</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col_names_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sortdict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;r&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> linewidth</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylim</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-1-numeric-feature-age">Example 1: Numeric feature: &quot;age&quot;<a href="#example-1-numeric-feature-age" class="hash-link" aria-label="Direct link to Example 1: Numeric feature: &quot;age&quot;" title="Direct link to Example 1: Numeric feature: &quot;age&quot;">​</a></h4><p>We can overlay the PDP on top of ICE plots. In the graph, the red line shows the PDP plot for the &quot;age&quot; feature, and the black lines show ICE plots for 50 randomly selected observations. </p><p>The visualization shows that all curves in the ICE plot follow a similar course. This means that the PDP (red line) is already a good summary of the relationships between the displayed feature &quot;age&quot; and the model&#x27;s average predictions of &quot;income&quot;. </p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_ice </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">age_df_pdp </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">overlay_ice_with_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">age_df_ice</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> age_df_pdp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:
 <img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_age_overlayed.png" alt="pdp_age_overlayed" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="example-2-categorical-feature-occupation">Example 2: Categorical feature: &quot;occupation&quot;<a href="#example-2-categorical-feature-occupation" class="hash-link" aria-label="Direct link to Example 2: Categorical feature: &quot;occupation&quot;" title="Direct link to Example 2: Categorical feature: &quot;occupation&quot;">​</a></h4><p>For visualization of categorical features, we are using a star plot.</p><ul><li>The X-axis here is a circle which is split into equal parts, each representing a feature value.</li><li>The Y-coordinate shows the dependence values. Each line represents a sample observation.</li></ul><p>Here we can see that &quot;Farming-fishing&quot; drives the least predictions - because values accumulated near the lowest probabilities, but, for example, &quot;Exec-managerial&quot; seems to have one of the highest impacts for model predictions.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">occupation_dep </span><span class="token operator">=</span><span class="token plain"> get_pandas_df_from_column</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_ice_categorical</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">occupation_dep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation_dependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp_occupation-star-plot.png" alt="pdp_occupation-star-plot" class="img_ev3q"></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conclusions-1">Conclusions<a href="#conclusions-1" class="hash-link" aria-label="Direct link to Conclusions" title="Direct link to Conclusions">​</a></h3><p>ICE plots show model behavior on individual observations. Each line represents the prediction from the model if we vary the feature of interest.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="pdp-based-feature-importance">PDP-based Feature Importance<a href="#pdp-based-feature-importance" class="hash-link" aria-label="Direct link to PDP-based Feature Importance" title="Direct link to PDP-based Feature Importance">​</a></h2><p>Using PDP we can calculate a simple partial dependence-based feature importance measure. We note that a  flat PDP indicates that varying the feature does not affect the prediction. The more the PDP varies, the more &quot;important&quot; the feature is. </p><p>If you want to learn more please check out <a href="https://christophm.github.io/interpretable-ml-book/pdp.html#pdp-based-feature-importance" target="_blank" rel="noopener noreferrer">Christoph M&#x27;s Interpretable ML Book</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="set-up-the-transformer-for-pdp-based-feature-importance">Set up the transformer for PDP-based Feature Importance<a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="hash-link" aria-label="Direct link to Set up the transformer for PDP-based Feature Importance" title="Direct link to Set up the transformer for PDP-based Feature Importance">​</a></h3><p>To plot PDP-based feature importance, we first need to set up the instance of <code>ICETransformer</code> by setting the <code>kind</code> parameter to <code>feature</code>. We can then call the <code>transform</code> function. </p><p><code>transform</code> returns a two-column table where the first columns are feature importance values and the second are corresponding features names. The rows are sorted in descending order by feature importance values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> ICETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    categoricalFeatures</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numericFeatures</span><span class="token operator">=</span><span class="token plain">numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">output_pdp_based_imp </span><span class="token operator">=</span><span class="token plain"> pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="visualization-2">Visualization<a href="#visualization-2" class="hash-link" aria-label="Direct link to Visualization" title="Direct link to Visualization">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Helper functions for visualization</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    values_list </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pdpBasedDependence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    names </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featureNames&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dependence_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> vec </span><span class="token keyword">in</span><span class="token plain"> values_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vec</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toArray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token plain">figsize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">names</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dependence_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Feature names&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> size</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PDP-based-feature-imporance&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This shows that the features <code>capital-gain</code> and <code>education-num</code> were the most important for the model, and <code>sex</code> and <code>education</code> were the least important.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">plot_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_pdp_based_imp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results will look like:</p><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/pdp-based-importance.png" alt="pdp_based-importance" class="img_ev3q"></p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="overall-conclusions">Overall conclusions<a href="#overall-conclusions" class="hash-link" aria-label="Direct link to Overall conclusions" title="Direct link to Overall conclusions">​</a></h2><p>Interpretation methods are very important responsible AI tools.</p><p>Partial dependence plots (PDP) and Individual Conditional Expectation (ICE) plots can be used to visualize and analyze interaction between the target response and a set of input features of interest.</p><p>PDPs show the dependence of the average prediction when varying each feature. In contrast, ICE shows the dependence for individual samples.  The approaches can help give rough estimates of a function&#x27;s deviation from a baseline. This is important not only to help debug and understand how a model behaves but is a useful step in building responsible AI systems. These methodologies can improve transparency and provide model consumers with an extra level of accountability by model creators.</p><p>Using examples above we showed how to calculate and visualize such plots at a scalable manner to understand how a classification or regression model makes predictions, which features heavily impact the model, and how model prediction changes when feature value changes.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Data Balance Analysis</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#partial-dependence-pdp-and-individual-conditional-expectation-ice-plots" class="table-of-contents__link toc-highlight">Partial Dependence (PDP) and Individual Conditional Expectation (ICE) plots</a><ul><li><a href="#read-and-prepare-the-dataset" class="table-of-contents__link toc-highlight">Read and prepare the dataset</a></li><li><a href="#fit-the-model-and-view-the-predictions" class="table-of-contents__link toc-highlight">Fit the model and view the predictions</a></li></ul></li><li><a href="#partial-dependence-plots" class="table-of-contents__link toc-highlight">Partial Dependence Plots</a><ul><li><a href="#set-up-the-transformer-for-pdp" class="table-of-contents__link toc-highlight">Set up the transformer for PDP</a></li><li><a href="#visualization" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#individual-conditional-expectation" class="table-of-contents__link toc-highlight">Individual Conditional Expectation</a><ul><li><a href="#set-up-the-transformer-for-ice" class="table-of-contents__link toc-highlight">Set up the transformer for ICE</a></li><li><a href="#visualization-1" class="table-of-contents__link toc-highlight">Visualization</a></li><li><a href="#conclusions-1" class="table-of-contents__link toc-highlight">Conclusions</a></li></ul></li><li><a href="#pdp-based-feature-importance" class="table-of-contents__link toc-highlight">PDP-based Feature Importance</a><ul><li><a href="#set-up-the-transformer-for-pdp-based-feature-importance" class="table-of-contents__link toc-highlight">Set up the transformer for PDP-based Feature Importance</a></li><li><a href="#visualization-2" class="table-of-contents__link toc-highlight">Visualization</a></li></ul></li><li><a href="#overall-conclusions" class="table-of-contents__link toc-highlight">Overall conclusions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html b/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html
index 924124c8b4..4500af4523 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Data Balance Analysis</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="data-balance-analysis-using-the-adult-census-income-dataset">Data Balance Analysis using the Adult Census Income dataset<a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="hash-link" aria-label="Direct link to Data Balance Analysis using the Adult Census Income dataset" title="Direct link to Data Balance Analysis using the Adult Census Income dataset">​</a></h2><p>In this example, we will conduct Data Balance Analysis (which consists on running three groups of measures) on the Adult Census Income dataset to determine how well features and feature values are represented in the dataset.</p><p>This dataset can be used to predict whether annual income exceeds $50,000/year or not based on demographic data from the 1994 U.S. Census. The dataset we&#x27;re reading contains 32,561 rows and 14 columns/features.</p><p><a href="https://archive.ics.uci.edu/ml/datasets/Adult" target="_blank" rel="noopener noreferrer">More info on the dataset here</a></p><hr><p>Data Balance Analysis consists of a combination of three groups of measures: Feature Balance Measures, Distribution Balance Measures, and Aggregate Balance Measures.
 In summary, Data Balance Analysis, when used as a step for building ML models, has the following benefits:</p><ul><li>It reduces costs of ML building through the early identification of data representation gaps that prompt data scientists to seek mitigation steps (such as collecting more data, following a specific sampling mechanism, creating synthetic data, and so on) before proceeding to train their models.</li><li>It enables easy end-to-end debugging of ML systems in combination with the <a href="https://responsibleaitoolbox.ai/responsible-ai-toolbox-capabilities/" target="_blank" rel="noopener noreferrer">RAI Toolbox</a> by providing a clear view of model-related issues versus data-related issues.</li></ul><hr><p>Note: If you are running this notebook in a Spark environment such as Azure Synapse or Databricks, then you can easily visualize the imbalance measures using the built-in plotting features.</p><p>Python dependencies:</p><div class="language-text codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">matplotlib==3.2.2</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numpy==1.19.2</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Convert the &quot;income&quot; column from {&lt;=50K, &gt;50K} to {0, 1} to represent our binary classification label column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">label_col </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;=50K&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="perform-preliminary-analysis-on-columns-of-interest">Perform preliminary analysis on columns of interest<a href="#perform-preliminary-analysis-on-columns-of-interest" class="hash-link" aria-label="Direct link to Perform preliminary analysis on columns of interest" title="Direct link to Perform preliminary analysis on columns of interest">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Choose columns/features to do data balance analysis on</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-feature-balance-measures"><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Calculate Feature Balance Measures</a><a href="#calculate-feature-balance-measures" class="hash-link" aria-label="Direct link to calculate-feature-balance-measures" title="Direct link to calculate-feature-balance-measures">​</a></h3><p>Feature Balance Measures allow us to see whether each combination of sensitive feature is receiving the positive outcome (true prediction) at equal rates.</p><p>In this context, we define a feature balance measure, also referred to as the parity, for label y as the absolute difference between the association metrics of two different sensitive classes <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mo stretchy="false">[</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo stretchy="false">]</mo></mrow><annotation encoding="application/x-tex">[x_A, x_B]</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mopen">[</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mclose">]</span></span></span></span></span>, with respect to the association metric <span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>i</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">A(x_i, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.31166399999999994em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">i</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span>. That is:</p><p><span class="math math-inline"><span class="katex"><span class="katex-mathml"><math xmlns="http://www.w3.org/1998/Math/MathML"><semantics><mrow><mi>p</mi><mi>a</mi><mi>r</mi><mi>i</mi><mi>t</mi><mi>y</mi><mo stretchy="false">(</mo><mi>y</mi><mi mathvariant="normal">∣</mi><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>A</mi><mo stretchy="false">(</mo><mo>⋅</mo><mo stretchy="false">)</mo><mo stretchy="false">)</mo><mo><mi mathvariant="normal">≔</mi></mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>A</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo><mo>−</mo><mi>A</mi><mo stretchy="false">(</mo><msub><mi>x</mi><mi>B</mi></msub><mo separator="true">,</mo><mi>y</mi><mo stretchy="false">)</mo></mrow><annotation encoding="application/x-tex">parity(y \vert x_A, x_B, A(\cdot)) \coloneqq A(x_A, y) - A(x_B, y)</annotation></semantics></math></span><span class="katex-html" aria-hidden="true"><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">p</span><span class="mord mathnormal">a</span><span class="mord mathnormal" style="margin-right:0.02778em">r</span><span class="mord mathnormal">i</span><span class="mord mathnormal">t</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mopen">(</span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mord">∣</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord">⋅</span><span class="mclose">)</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2777777777777778em"></span><span class="mrel"><span class="mrel"><span class="mop" style="position:relative;top:-0.03472em">:</span></span><span class="mrel"><span class="mspace" style="margin-right:-0.06666666666666667em"></span></span><span class="mrel">=</span></span><span class="mspace" style="margin-right:0.2777777777777778em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight">A</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span><span class="mspace" style="margin-right:0.2222222222222222em"></span><span class="mbin">−</span><span class="mspace" style="margin-right:0.2222222222222222em"></span></span><span class="base"><span class="strut" style="height:1em;vertical-align:-0.25em"></span><span class="mord mathnormal">A</span><span class="mopen">(</span><span class="mord"><span class="mord mathnormal">x</span><span class="msupsub"><span class="vlist-t vlist-t2"><span class="vlist-r"><span class="vlist" style="height:0.32833099999999993em"><span style="top:-2.5500000000000003em;margin-left:0em;margin-right:0.05em"><span class="pstrut" style="height:2.7em"></span><span class="sizing reset-size6 size3 mtight"><span class="mord mathnormal mtight" style="margin-right:0.05017em">B</span></span></span></span><span class="vlist-s">​</span></span><span class="vlist-r"><span class="vlist" style="height:0.15em"><span></span></span></span></span></span></span><span class="mpunct">,</span><span class="mspace" style="margin-right:0.16666666666666666em"></span><span class="mord mathnormal" style="margin-right:0.03588em">y</span><span class="mclose">)</span></span></span></span></span></p><p>Using the dataset, we can see if the various sexes and races are receiving &gt;50k income at equal or unequal rates.</p><p>Note: Many of these metrics were influenced by this paper <a href="https://arxiv.org/abs/2103.03417" target="_blank" rel="noopener noreferrer">Measuring Model Biases in the Absence of Ground Truth</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> FeatureBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    FeatureBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">label_col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVerbose</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by Statistical Parity descending for all features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;sex&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Drill down to feature == &quot;race&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-feature-balance-measures">Visualize Feature Balance Measures<a href="#visualize-feature-balance-measures" class="hash-link" aria-label="Direct link to Visualize Feature Balance Measures" title="Direct link to Visualize Feature Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">races </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_rows </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">filter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureBalanceMeasure.dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ClassB&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> dp_rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_dp_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">zeros</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_value </span><span class="token keyword">in</span><span class="token plain"> race_dp_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j </span><span class="token operator">=</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_a</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">class_b</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    dp_value </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dp_value</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> dp_value</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">colormap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RdBu&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> dp_max </span><span class="token operator">=</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">im </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmin</span><span class="token operator">=</span><span class="token plain">dp_min</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vmax</span><span class="token operator">=</span><span class="token plain">dp_max</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> cmap</span><span class="token operator">=</span><span class="token plain">colormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colorbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">im</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax</span><span class="token operator">=</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cbar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_yticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anchor&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> j </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">races</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        text </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_dp_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> j</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;k&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Statistical Parity of Races in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_RacesDP.png" alt="Statistical Parity of Races in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-feature-balance-measures">Interpret Feature Balance Measures<a href="#interpret-feature-balance-measures" class="hash-link" aria-label="Direct link to Interpret Feature Balance Measures" title="Direct link to Interpret Feature Balance Measures">​</a></h4><p>Statistical Parity:</p><ul><li>When it is positive, it means that ClassA sees the positive outcome more than ClassB.</li><li>When it is negative, it means that ClassB sees the positive outcome more than ClassA.</li></ul><hr><p>From the results, we can tell the following:</p><p>For Sex:</p><ul><li>SP(Male, Female) = 0.1963 shows &quot;Male&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Female&quot; observations.</li></ul><p>For Race:</p><ul><li>SP(Other, Asian-Pac-Islander) = -0.1734 shows &quot;Other&quot; observations are associated with &quot;&gt;50k&quot; income label less than &quot;Asian-Pac-Islander&quot; observations.</li><li>SP(White, Other) = 0.1636 shows &quot;White&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Other&quot; observations.</li><li>SP(Asian-Pac-Islander, Amer-Indian-Eskimo) = 0.1494 shows &quot;Asian-Pac-Islander&quot; observations are associated with &quot;&gt;50k&quot; income label more often than &quot;Amer-Indian-Eskimo&quot; observations.</li></ul><p>Again, you can take mitigation steps to upsample/downsample your data to be less biased towards certain features and feature values.</p><p>Built-in mitigation steps are coming soon.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-distribution-balance-measures">Calculate <a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Distribution Balance Measures</a><a href="#calculate-distribution-balance-measures" class="hash-link" aria-label="Direct link to calculate-distribution-balance-measures" title="Direct link to calculate-distribution-balance-measures">​</a></h3><p>Distribution Balance Measures allow us to compare our data with a reference distribution (i.e. uniform distribution). They are calculated per sensitive column and don&#x27;t use the label column.                          |</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> DistributionBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    DistributionBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Sort by JS Distance descending</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">abs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure.js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">desc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="visualize-distribution-balance-measures">Visualize Distribution Balance Measures<a href="#visualize-distribution-balance-measures" class="hash-link" aria-label="Direct link to Visualize Distribution Balance Measures" title="Direct link to Visualize Distribution Balance Measures">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">distribution_rows </span><span class="token operator">=</span><span class="token plain"> distribution_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_row </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> distribution_rows </span><span class="token keyword">if</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;FeatureName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">==</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;DistributionBalanceMeasure&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">measures_of_interest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;kl_divergence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;js_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inf_norm_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;total_variation_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasserstein_dist&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">race_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">race_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sex_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token builtin" style="color:rgb(174, 129, 255)">round</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sex_row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">measure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> measure </span><span class="token keyword">in</span><span class="token plain"> measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">x </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">arange</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">width </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ax </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects1 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">-</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> race_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rects2 </span><span class="token operator">=</span><span class="token plain"> ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x </span><span class="token operator">+</span><span class="token plain"> width </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sex_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Measure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Distribution Balance Measures of Sex and Race in Adult Dataset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">measures_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">legend</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_xticklabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;right&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> rotation_mode</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;default&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> rect </span><span class="token keyword">in</span><span class="token plain"> rects</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        height </span><span class="token operator">=</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ax</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">annotate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xy</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> rect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_width</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> height</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xytext</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># 1 point vertical offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            textcoords</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset points&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ha</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;center&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            va</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bottom&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">autolabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rects2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/responsible_ai/DataBalanceAnalysis_AdultCensusIncome_DistributionMeasures.png" alt="Distribution Balance Measures of Sex and Race in Adult Dataset" class="img_ev3q"></p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-distribution-balance-measures">Interpret Distribution Balance Measures<a href="#interpret-distribution-balance-measures" class="hash-link" aria-label="Direct link to Interpret Distribution Balance Measures" title="Direct link to Interpret Distribution Balance Measures">​</a></h4><p>Race has a JS Distance of 0.5104 while Sex has a JS Distance of 0.1217.</p><p>Knowing that JS Distance is between <!-- -->[0, 1]<!-- --> where 0 means perfectly balanced distribution, we can tell that:</p><ul><li>There is a larger disparity between various races than various sexes in our dataset.</li><li>Race is nowhere close to a perfectly balanced distribution (i.e. some races are seen ALOT more than others in our dataset).</li><li>Sex is fairly close to a perfectly balanced distribution.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="calculate-aggregate-balance-measures">Calculate <a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Aggregate Balance Measures</a><a href="#calculate-aggregate-balance-measures" class="hash-link" aria-label="Direct link to calculate-aggregate-balance-measures" title="Direct link to calculate-aggregate-balance-measures">​</a></h3><p>Aggregate Balance Measures allow us to obtain a higher notion of inequality. They are calculated on the global set of sensitive columns and don&#x27;t use the label column.</p><p>These measures look at distribution of records across all combinations of sensitive columns. For example, if Sex and Race are sensitive columns, it shall try to quantify imbalance across all combinations - (Male, Black), (Female, White), (Male, Asian-Pac-Islander), etc.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">exploratory </span><span class="token keyword">import</span><span class="token plain"> AggregateBalanceMeasure</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">aggregate_balance_measures </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AggregateBalanceMeasure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSensitiveCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cols_of_interest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">aggregate_balance_measures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="interpret-aggregate-balance-measures">Interpret Aggregate Balance Measures<a href="#interpret-aggregate-balance-measures" class="hash-link" aria-label="Direct link to Interpret Aggregate Balance Measures" title="Direct link to Interpret Aggregate Balance Measures">​</a></h4><p>An Atkinson Index of 0.7779 lets us know that 77.79% of data points need to be foregone to have a more equal share among our features.</p><p>It lets us know that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</p><ul><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summary">Summary<a href="#summary" class="hash-link" aria-label="Direct link to Summary" title="Direct link to Summary">​</a></h3><p>Throughout the course of this sample notebook, we have:</p><ol><li>Chosen &quot;Race&quot; and &quot;Sex&quot; as columns of interest in the Adult Census Income dataset.</li><li>Done preliminary analysis on our dataset. </li><li>Ran the 3 groups of measures that compose our <strong>Data Balance Analysis</strong>:</li></ol><ul><li><strong>Feature Balance Measures</strong><ul><li>Calculated Feature Balance Measures to see that the highest Statistical Parity is in &quot;Sex&quot;: Males see &gt;50k income much more than Females.</li><li>Visualized Statistical Parity of Races to see that Asian-Pac-Islander sees &gt;50k income much more than Other, in addition to other race combinations.</li></ul></li><li><strong>Distribution Balance Measures</strong> <ul><li>Calculated Distribution Balance Measures to see that &quot;Sex&quot; is much closer to a perfectly balanced distribution than &quot;Race&quot;.</li><li>Visualized various distribution balance measures to compare their values for &quot;Race&quot; and &quot;Sex&quot;.</li></ul></li><li><strong>Aggregate Balance Measures</strong><ul><li>Calculated Aggregate Balance Measures to see that we need to forego 77.79% of data points to have a perfectly balanced dataset. We identified that our dataset is leaning towards maximum inequality, and we should take actionable steps to:</li><li>Upsample data points where the feature value is barely observed.</li><li>Downsample data points where the feature value is observed much more than others.</li></ul></li></ul><p><strong>In conclusion:</strong></p><ul><li>These measures provide an indicator of disparity on the data, allowing for users to explore potential mitigations before proceeding to train. </li><li>Users can use these measures to set thresholds on their level of &quot;tolerance&quot; for data representation.</li><li>Production pipelines can use these measures as baseline for models that require frequent retraining on new data. </li><li>These measures can also be saved as key metadata for the model/service built and added as part of model cards or transparency notes helping drive overall accountability for the ML service built and its performance across different demographics or sensitive attributes.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Explanation Dashboard</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Snow Leopard Detection</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#data-balance-analysis-using-the-adult-census-income-dataset" class="table-of-contents__link toc-highlight">Data Balance Analysis using the Adult Census Income dataset</a><ul><li><a href="#perform-preliminary-analysis-on-columns-of-interest" class="table-of-contents__link toc-highlight">Perform preliminary analysis on columns of interest</a></li><li><a href="#calculate-feature-balance-measures" class="table-of-contents__link toc-highlight">Calculate Feature Balance Measures</a></li><li><a href="#calculate-distribution-balance-measures" class="table-of-contents__link toc-highlight">Calculate Distribution Balance Measures</a></li><li><a href="#calculate-aggregate-balance-measures" class="table-of-contents__link toc-highlight">Calculate Aggregate Balance Measures</a></li><li><a href="#summary" class="table-of-contents__link toc-highlight">Summary</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html b/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html
index 16a750177e..20ef2b904f 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Snow Leopard Detection</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automated-snow-leopard-detection-with-synapse-machine-learning">Automated Snow Leopard Detection with Synapse Machine Learning<a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="hash-link" aria-label="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning" title="Direct link to Automated Snow Leopard Detection with Synapse Machine Learning">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLTrust.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bing_search_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># WARNING this notebook requires a lot of memory.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># If you get a heap space error, try dropping the number of images bing returns</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># or by writing out the images to parquet first</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark </span><span class="token keyword">import</span><span class="token plain"> FluentAPI</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">offset </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    parameters </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">query</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> offset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> offset </span><span class="token keyword">in</span><span class="token plain"> offsets </span><span class="token keyword">for</span><span class="token plain"> query </span><span class="token keyword">in</span><span class="token plain"> queries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">parameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Apply Bing Image Search</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Set the API Key</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offsets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;queries&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a column containing the query words</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify the number of images to return per offset</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;photo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify a filter to ensure we get photos</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SparkSummit2/cog_services.png" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> n</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> image_cols</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">n</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;th&gt;&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> c </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/th&gt;&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> c </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    style </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;!DOCTYPE html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;head&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="display:inline-block;color:rgb(230, 219, 116)"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/head&gt;&quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    table </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">if</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> image_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;&lt;img src=&quot;{}&quot;,  width=&quot;100&quot;&gt;&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                rep </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;td&gt;{}&lt;/td&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rep</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">append</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;/tr&gt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tableHTML </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">table</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    body </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    {} </span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &lt;/tr&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  {}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/table&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/body&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&lt;/html&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">  &quot;&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        header</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tableHTML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> running_on_databricks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            displayHTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">else</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">import</span><span class="token plain"> IPython</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            IPython</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">HTML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">style </span><span class="token operator">+</span><span class="token plain"> body</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">except</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">pass</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardQueries </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">snowLeopardUrls </span><span class="token operator">=</span><span class="token plain"> bingPhotoSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;snow leopard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> snowLeopardQueries</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> pages</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/random_words.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomLinks </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    randomWords</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bing_search_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQueryCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;other&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">displayDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">images </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    snowLeopardUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">union</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randomLinks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">distinct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mlTransform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">downloadFromUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;urls&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> concurrency</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> timeout</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropna</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> images</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ImageFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> UDFTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ImageFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            autoConvertToColor</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            ignoreDecodingErrors</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ResNet50&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">maxIter</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> regParam</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getIndex</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fitModel </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/SLPipeline.PNG" width="900" class="img_ev3q"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot </span><span class="token keyword">import</span><span class="token plain"> confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">figure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> classLabels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plotConfusionMatrix</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> ImageLIME</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_image_url </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/snow_leopard1.jpg&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> urllib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">urlopen</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_image_url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">as</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    barr </span><span class="token operator">=</span><span class="token plain"> url</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_subsample </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">bytearray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">barr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">fitModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leopard_prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">300</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result </span><span class="token operator">=</span><span class="token plain"> lime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_subsample</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_bytes </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    superpixels </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;clusters&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    weights </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mean_weight </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">percentile</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    img </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PIL</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Image</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">open</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">BytesIO</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_bytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">convert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;RGBA&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    image_array </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">img</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">copy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> w</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weights</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">if</span><span class="token plain"> w </span><span class="token operator">&gt;</span><span class="token plain"> mean_weight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token keyword">for</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">in</span><span class="token plain"> sp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">y</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">clf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">imshow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">image_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Gets first row from the LIME-transformed data frame</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    plot_superpixels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="your-results-will-look-like">Your results will look like:<a href="#your-results-will-look-like" class="hash-link" aria-label="Direct link to Your results will look like:" title="Direct link to Your results will look like:">​</a></h3><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/SnowLeopardAD/lime_results.png" width="900" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Data Balance Analysis</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automated-snow-leopard-detection-with-synapse-machine-learning" class="table-of-contents__link toc-highlight">Automated Snow Leopard Detection with Synapse Machine Learning</a><ul><li><a href="#your-results-will-look-like" class="table-of-contents__link toc-highlight">Your results will look like:</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/index.html b/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/index.html
index f0006265f6..cde908eb82 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/index.html	
@@ -16,14 +16,14 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Tabular Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Tabular Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Tabular Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Tabular Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="theme-doc-markdown markdown"><h1>Interpretability - Tabular SHAP explainer</h1><p>In this example, we use Kernel SHAP to explain a tabular classification model built from the Adults Census dataset.</p><p>First we import the packages and define some UDFs we need later.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">float</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">v</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Now let&#x27;s read the data and train a binary classification model.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">labelIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> stringOrderType</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;alphabetAsc&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label index assigment: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">set</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">labels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">training </span><span class="token operator">=</span><span class="token plain"> labelIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_idx </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_idx&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">categorical_features_enc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">col </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_enc&quot;</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> col </span><span class="token keyword">in</span><span class="token plain"> categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">numeric_features </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">strIndexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onehotEnc </span><span class="token operator">=</span><span class="token plain"> OneHotEncoder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_idx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectAssem </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features_enc </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lr </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> weightCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">strIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> onehotEnc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vectAssem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>After the model is trained, we randomly select some observations to be explained.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We create a TabularSHAP explainer, set the input columns to all the features the model takes, specify the model and the target output column we&#x27;re trying to explain. In this case, we&#x27;re trying to explain the &quot;probability&quot; output, which is a vector of length 2, and we&#x27;re only looking at class 1 probability. Specify targetClasses to <code>[0, 1]</code> if you want to explain class 0 and 1 probability at the same time. Finally we sample 100 rows from the training data for background data, which is used for integrating out features in Kernel SHAP.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    backgroundData</span><span class="token operator">=</span><span class="token plain">broadcast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">training</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">orderBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_df </span><span class="token operator">=</span><span class="token plain"> shap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">explain_instances</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Once we have the resulting dataframe, we extract the class 1 probability of the model output, the SHAP values for the target class, the original features and the true label. Then we convert it to a pandas dataframe for visualization.
 For each observation, the first element in the SHAP values vector is the base value (the mean output of the background dataset), and each of the following element is the SHAP values for each feature.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vec_access</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vector_to_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local </span><span class="token operator">=</span><span class="token plain"> shaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sort_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ascending</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> inplace</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ignore_index</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;display.max_colwidth&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shaps_local</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We use plotly subplot to visualize the SHAP values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots </span><span class="token keyword">import</span><span class="token plain"> make_subplots</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> plotly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">graph_objects </span><span class="token keyword">as</span><span class="token plain"> go</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> categorical_features </span><span class="token operator">+</span><span class="token plain"> numeric_features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">features_with_base </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Base&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> features</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rows </span><span class="token operator">=</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig </span><span class="token operator">=</span><span class="token plain"> make_subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    rows</span><span class="token operator">=</span><span class="token plain">rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    cols</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subplot_titles</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Probability: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">apply</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;{:.2%}&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;; Label: &quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">+</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> index</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> shaps_local</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">iterrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_values </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feature</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> feature </span><span class="token keyword">in</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_values </span><span class="token operator">=</span><span class="token plain"> row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    list_of_tuples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">zip</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features_with_base</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> shap_values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap_pdf </span><span class="token operator">=</span><span class="token plain"> pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">list_of_tuples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add_trace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        go</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Bar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            x</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            y</span><span class="token operator">=</span><span class="token plain">shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            hovertext</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> shap_pdf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">astype</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        row</span><span class="token operator">=</span><span class="token plain">index </span><span class="token operator">+</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        col</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_yaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> zerolinecolor</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_xaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">type</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;category&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> tickangle</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> fixedrange</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">update_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">400</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> rows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> title_text</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SHAP explanations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">if</span><span class="token plain"> </span><span class="token keyword">not</span><span class="token plain"> running_on_synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your results should look like:</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/explainers/tabular-shap.png" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Interpreting Model Predictions</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Text Explainers</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Responsible AI/Text Explainers/index.html b/docs/next/Explore Algorithms/Responsible AI/Text Explainers/index.html
index 23595225d8..ead5c0b39e 100644
--- a/docs/next/Explore Algorithms/Responsible AI/Text Explainers/index.html	
+++ b/docs/next/Explore Algorithms/Responsible AI/Text Explainers/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Responsible AI/Text Explainers/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Responsible AI/Text Explainers/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Responsible AI/Text Explainers/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Responsible AI/Text Explainers/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Responsible AI/Text Explainers/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Interpreting Model Predictions</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/">Tabular Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Text Explainers/">Text Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/">Image Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/PDP and ICE Explainers/">PDP and ICE Explainers</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Data Balance Analysis/">Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Explanation Dashboard/">Explanation Dashboard</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Data Balance Analysis/">Quickstart - Data Balance Analysis</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Quickstart - Snow Leopard Detection/">Quickstart - Snow Leopard Detection</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Responsible AI/Text Explainers/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Responsible AI</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Text Explainers</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="interpretability---text-explainers">Interpretability - Text Explainers<a href="#interpretability---text-explainers" class="hash-link" aria-label="Direct link to Interpretability - Text Explainers" title="Direct link to Interpretability - Text Explainers">​</a></h2><p>In this example, we use LIME and Kernel SHAP explainers to explain a text classification model.</p><p>First we import the packages and define some UDFs and a plotting function we will need later.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.functions import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.sql.types import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml import Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.classification import LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from pyspark.ml.functions import vector_to_array</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.explainers import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.featurize.text import TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">from synapse.ml.core.platform import *</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vec_access = udf(lambda v, i: float(v[i]), FloatType())</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Load training data, and convert rating to binary label.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark.read.parquet(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;label&quot;, (col(&quot;rating&quot;) &gt; 3).cast(LongType()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;label&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(data)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>We train a text classification model, and randomly sample 10 rows to explain.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train, test = data.randomSplit([0.60, 0.40])</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline = Pipeline(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages=[</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol=&quot;features&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useStopWordsRemover=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            useIDF=True,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            minDocFreq=20,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            numFeatures=1 &lt;&lt; 16,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LogisticRegression(maxIter=100, regParam=0.005, labelCol=&quot;label&quot;, featuresCol=&quot;features&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model = pipeline.fit(train)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction = model.transform(test)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explain_instances = prediction.orderBy(rand()).limit(10)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">def plotConfusionMatrix(df, label, prediction, classLabels):</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    from synapse.ml.plot import confusionMatrix</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    import matplotlib.pyplot as plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    fig = plt.figure(figsize=(4.5, 4.5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    confusionMatrix(df, label, prediction, classLabels)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    if running_on_synapse():</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        plt.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    else:</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        display(fig)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plotConfusionMatrix(model.transform(test), &quot;label&quot;, &quot;prediction&quot;, [0, 1])</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>First we use the LIME text explainer to explain the model&#x27;s predicted probability for a given observation.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime = TextLIME(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;weights&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    samplingFraction=0.7,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=2000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lime.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;weights&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;weights&quot;, vector_to_array(col(&quot;weights&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_weights&quot;, arrays_zip(&quot;tokens&quot;, &quot;weights&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(lime_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_weights&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Then we use the Kernel SHAP text explainer to explain the model&#x27;s predicted probability for a given observation.</p><blockquote><p>Notice that we drop the base value from the SHAP output before displaying the SHAP values. The base value is the model output for an empty string.</p></blockquote><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap = TextSHAP(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    model=model,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol=&quot;shaps&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCol=&quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetCol=&quot;probability&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    targetClasses=[1],</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    tokensCol=&quot;tokens&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numSamples=5000,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap_results = (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    shap.transform(explain_instances)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .select(&quot;tokens&quot;, &quot;shaps&quot;, &quot;r2&quot;, &quot;probability&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;probability&quot;, vec_access(&quot;probability&quot;, lit(1)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, vector_to_array(col(&quot;shaps&quot;).getItem(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;shaps&quot;, slice(col(&quot;shaps&quot;), lit(2), size(col(&quot;shaps&quot;))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;r2&quot;, vec_access(&quot;r2&quot;, lit(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .withColumn(&quot;tokens_shaps&quot;, arrays_zip(&quot;tokens&quot;, &quot;shaps&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display(shap_results.select(&quot;probability&quot;, &quot;r2&quot;, &quot;tokens_shaps&quot;, &quot;text&quot;).orderBy(col(&quot;probability&quot;).desc()))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Tabular Explainers/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Tabular Explainers</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Image Explainers/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Image Explainers</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interpretability---text-explainers" class="table-of-contents__link toc-highlight">Interpretability - Text Explainers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html
index 0b2761ccd0..78cc0e6084 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Contextual Bandits</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Contextual-Bandits using Vowpal Wabbit</h1><p>In the contextual bandit problem, a learner repeatedly observes a context, chooses an action, and observes a loss/cost/reward for the chosen action only. Contextual bandit algorithms use additional side information (or context) to aid real world decision-making. They work well for choosing actions in dynamic environments where options change rapidly, and the set of available actions is limited.</p><p>An in-depth tutorial can be found <a href="https://vowpalwabbit.org/docs/vowpal_wabbit/python/latest/tutorials/python_Contextual_bandits_and_Vowpal_Wabbit.html" target="_blank" rel="noopener noreferrer">here</a></p><p><a href="https://azure.microsoft.com/en-us/products/cognitive-services/personalizer" target="_blank" rel="noopener noreferrer">Azure Personalizer</a> emits logs in DSJSON-format. This example demonstrates how to perform off-policy evaluation.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step1-read-the-dataset">Step1: Read the dataset<a href="#step1-read-the-dataset" class="hash-link" aria-label="Direct link to Step1: Read the dataset" title="Direct link to Step1: Read the dataset">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/decisionservice.json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector<a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Step 2: Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitDSJsonTransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitDSJsonTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDsJsonColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show structured nature of rewards</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># exclude JSON to avoid overflow</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-3-train-model">Step 3: Train model<a href="#step-3-train-model" class="hash-link" aria-label="Direct link to Step 3: Train model" title="Direct link to Step 3: Train model">​</a></h2><p>VowpalWabbitGeneric performs these steps:</p><ul><li>trains a model for each split (=group)</li><li>synchronizes accross partitions after every split</li><li>store the 1-step ahead predictions in the model</li></ul><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_adf --cb_type mtr --clip_p 0.1 -q GT -q MS -q GR -q OT -q MT -q OS --dsjson --preserve_performance_counters&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSplitCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;splitId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="step-4-predict-and-evaluate">Step 4: Predict and evaluate<a href="#step-4-predict-and-evaluate" class="hash-link" aria-label="Direct link to Step 4: Predict and evaluate" title="Direct link to Step 4: Predict and evaluate">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOneStepAheadPredictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># .show(5, False)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers </span><span class="token operator">=</span><span class="token plain"> df_train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_headers_predictions </span><span class="token operator">=</span><span class="token plain"> df_headers</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EventId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitCSETransformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitCSETransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_headers_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>For each field of the reward column the metrics are calculated</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">per_reward_metrics </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;reward.*&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">per_reward_metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Multi-class classification</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#step1-read-the-dataset" class="table-of-contents__link toc-highlight">Step1: Read the dataset</a></li><li><a href="#step-2-use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="table-of-contents__link toc-highlight">Step 2: Use VowpalWabbitFeaturizer to convert data features into vector</a></li><li><a href="#step-3-train-model" class="table-of-contents__link toc-highlight">Step 3: Train model</a></li><li><a href="#step-4-predict-and-evaluate" class="table-of-contents__link toc-highlight">Step 4: Predict and evaluate</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html
index 8d204e9390..b6e0efe8e9 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Multi-class classification</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Multi-class Classification using Vowpal Wabbit</h1><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/iris.txt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> StringIndexer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexer </span><span class="token operator">=</span><span class="token plain"> StringIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;variety&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sepal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_length&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;petal_width&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># label needs to be integer (0 to n)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_label </span><span class="token operator">=</span><span class="token plain"> indexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># featurize data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_featurized </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_label</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df_featurized</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --oaa 3 --holdout_off --loss_function=logistic --indexing 0 -q ::&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">About</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Contextual Bandits</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/index.html
index 5b8fd735eb..fe98cdf1e3 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -47,7 +47,7 @@
 Similar to LightGBM all training instances must be running at the same time, thus the maximum parallelism is restricted by the
 number of executors available in the cluster. Under the hood, VW&#x27;s built-in spanning tree functionality is used to coordinate <em>allreduce</em>.
 Required parameters are automatically determined and supplied to VW. The spanning tree coordination process is run on the driver node.</p><ul><li>Pro: seamless parallelization.</li><li>Cons: currently barrier execution mode isn&#x27;t implemented and thus if one node crashes the complete job needs to be manually restarted.</li></ul></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Image Transformations</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Multi-class classification</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#overview" class="table-of-contents__link toc-highlight">Overview</a></li><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#usage" class="table-of-contents__link toc-highlight">Usage</a></li><li><a href="#hyper-parameter-tuning" class="table-of-contents__link toc-highlight">Hyper-parameter tuning</a></li><li><a href="#architecture" class="table-of-contents__link toc-highlight">Architecture</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html
index 18954e2010..3a06c520a0 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using SparkML Vectors</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification, Quantile Regression, and Regression</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using VW-native Format</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html
index e61a5157c0..a025ba730b 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">LightGBM</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/AI Services/Overview/">AI Services</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenAI/Langchain/">OpenAI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Deep Learning/Getting Started/">Deep Learning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Responsible AI</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Causal Inference/Overview/">Causal Inference</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Classification/Quickstart - Train Classifier/">Classification</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Regression/Quickstart - Data Cleaning/">Regression</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Anomaly Detection/Quickstart - Isolation Forests/">Anomaly Detection</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Hyperparameter Tuning/HyperOpt/">Hyperparameter Tuning</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/OpenCV/Image Transformations/">OpenCV</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">Vowpal Wabbit</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Overview/">About</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Multi-class classification/">Multi-class classification</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/">Contextual Bandits</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/">Quickstart - Classification, Quantile Regression, and Regression</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/">Quickstart - Classification using SparkML Vectors</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-3 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">Quickstart - Classification using VW-native Format</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-2 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" tabindex="0" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/">Other Algorithms</a></div></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using VW-native Format/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Explore Algorithms</span><meta itemprop="position" content="1"></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Vowpal Wabbit</span><meta itemprop="position" content="2"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Classification using VW-native Format</span><meta itemprop="position" content="3"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><img loading="lazy" width="200" src="https://mmlspark.blob.core.windows.net/graphics/emails/vw-blue-dark-orange.svg" class="img_ev3q"><h1>Binary Classification with VowpalWabbit on Criteo Dataset</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="sparkml-vector-input">SparkML Vector input<a href="#sparkml-vector-input" class="hash-link" aria-label="Direct link to SparkML Vector input" title="Direct link to SparkML Vector input">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">as</span><span class="token plain"> T</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql </span><span class="token keyword">import</span><span class="token plain"> functions </span><span class="token keyword">as</span><span class="token plain"> F</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> T</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;delimiter&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;\t&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/criteo_day0_1k.csv.gz&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="reformat-into-vw-native-format">Reformat into VW-native format<a href="#reformat-into-vw-native-format" class="hash-link" aria-label="Direct link to Reformat into VW-native format" title="Direct link to Reformat into VW-native format">​</a></h4><p>See VW <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Input-format" target="_blank" rel="noopener noreferrer">docs</a> for format details</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># create VW string format</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;|&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;i&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">13</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token operator">*</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">concat_ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot; &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain">cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitGeneric</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># number of partitions determines data parallelism</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitGeneric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    useBarrierExecutionMode</span><span class="token operator">=</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function logistic --link logistic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> F</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">substring</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Smart Adaptive Recommendations/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">SAR Algorithm</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#sparkml-vector-input" class="table-of-contents__link toc-highlight">SparkML Vector input</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html
index 9a1de83756..ec22bf30d8 100644
--- a/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
+++ b/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification, Quantile Regression, and Regression/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -38,7 +38,7 @@
 We&#x27;re working on creating a more portable version by statically linking Boost and lib C++.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbit-usage">VowpalWabbit Usage:<a href="#vowpalwabbit-usage" class="hash-link" aria-label="Direct link to VowpalWabbit Usage:" title="Direct link to VowpalWabbit Usage:">​</a></h3><ul><li>VowpalWabbitClassifier: used to build classification models.</li><li>VowpalWabbitRegressor: used to build regression models.</li><li>VowpalWabbitFeaturizer: used for feature hashing and extraction. For details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Feature-Hashing-and-Extraction" target="_blank" rel="noopener noreferrer">here</a>.</li><li>VowpalWabbitContextualBandit: used to solve contextual bandits problems. For algorithm details please visit <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Contextual-Bandit-algorithms" target="_blank" rel="noopener noreferrer">here</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="heart-disease-detection-with-vowpalwabbit-classifier">Heart Disease Detection with VowpalWabbit Classifier<a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="hash-link" aria-label="Direct link to Heart Disease Detection with VowpalWabbit Classifier" title="Direct link to Heart Disease Detection with VowpalWabbit Classifier">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/heart disease.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset">Read dataset<a href="#read-dataset" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;csv&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;header&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/heart_disease_prediction_data.csv&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># print dataset basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-the-dataset-into-train-and-test">Split the dataset into train and test<a href="#split-the-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split the dataset into train and test" title="Direct link to Split the dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector">Use VowpalWabbitFeaturizer to convert data features into vector<a href="#use-vowpalwabbitfeaturizer-to-convert-data-features-into-vector" class="hash-link" aria-label="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector" title="Direct link to Use VowpalWabbitFeaturizer to convert data features into vector">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_data </span><span class="token operator">=</span><span class="token plain"> featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training">Model Training<a href="#model-training" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction">Model Prediction<a href="#model-prediction" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">predictions </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="adult-census-with-vowpalwabbitclassifier">Adult Census with VowpalWabbitClassifier<a href="#adult-census-with-vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to Adult Census with VowpalWabbitClassifier" title="Direct link to Adult Census with VowpalWabbitClassifier">​</a></h2><p>In this example, we predict incomes from the Adult Census dataset using Vowpal Wabbit (VW) Classifier in SynapseML.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-and-split-them-into-train--test">Read dataset and split them into train &amp; test<a href="#read-dataset-and-split-them-into-train--test" class="hash-link" aria-label="Direct link to Read dataset and split them into train &amp; test" title="Direct link to Read dataset and split them into train &amp; test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-1">Model Training<a href="#model-training-1" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><p>We define a pipeline that includes feature engineering and training of a VW classifier. We use a featurizer provided by VW that hashes the feature names. Note that VW expects classification labels being -1 or 1. Thus, the income category is mapped to this space before feeding training data into the pipeline.</p><p>Note: VW supports distributed learning, and it&#x27;s controlled by number of partitions of dataset.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Define classification label</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">train </span><span class="token operator">=</span><span class="token plain"> train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Specify featurizer</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: &quot;passThroughArgs&quot; parameter lets you pass in any params not exposed through our API. Full command line argument docs can be found <a href="https://github.com/VowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">here</a>.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Define VW classification model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--loss_function=logistic --quiet --holdout_off&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Create a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_trained </span><span class="token operator">=</span><span class="token plain"> vw_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-1">Model Prediction<a href="#model-prediction-1" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><p>After the model is trained, we apply it to predict the income of each sample in the test set.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># Making predictions</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test </span><span class="token operator">=</span><span class="token plain"> test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> when</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">prediction </span><span class="token operator">=</span><span class="token plain"> vw_trained</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Finally, we evaluate the model performance using ComputeModelStatistics function which will compute confusion matrix, accuracy, precision, recall, and AUC by default for classification models.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression">California house price prediction with VowpalWabbitRegressor - Quantile Regression<a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="hash-link" aria-label="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression" title="Direct link to California house price prediction with VowpalWabbitRegressor - Quantile Regression">​</a></h2><p>In this example, we show how to build regression model with VW using California housing dataset</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-1">Read dataset<a href="#read-dataset-1" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><p>We use <a href="https://scikit-learn.org/stable/datasets/real_world.html#california-housing-dataset" target="_blank" rel="noopener noreferrer"><em>California Housing</em> dataset</a>.
 The data was derived from the 1990 U.S. census. It consists of 20640 entries with 8 features.
 We use <code>sklearn.datasets</code> module to download it easily, then split the set into training and testing by 75/25.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> math</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">colors </span><span class="token keyword">import</span><span class="token plain"> ListedColormap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Normalize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cm </span><span class="token keyword">import</span><span class="token plain"> get_cmap</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> matplotlib</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyplot </span><span class="token keyword">as</span><span class="token plain"> plt</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> VowpalWabbitFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> numpy </span><span class="token keyword">as</span><span class="token plain"> np</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> pandas </span><span class="token keyword">as</span><span class="token plain"> pd</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> sklearn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">datasets </span><span class="token keyword">import</span><span class="token plain"> fetch_california_housing</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">california </span><span class="token operator">=</span><span class="token plain"> fetch_california_housing</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feature_cols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;f&quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">shape</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">header </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> feature_cols</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    pd</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        data</span><span class="token operator">=</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">column_stack</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> california</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> columns</span><span class="token operator">=</span><span class="token plain">header</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dataframe has {} rows&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test_data </span><span class="token operator">=</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">42</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Exploratory analysis: plot feature distributions over different target values.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">features </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">values </span><span class="token operator">=</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">drop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ncols </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">nrows </span><span class="token operator">=</span><span class="token plain"> math</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ceil</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">/</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">yy </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">r</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> r </span><span class="token keyword">in</span><span class="token plain"> train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axes </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> sharey</span><span class="token operator">=</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">for</span><span class="token plain"> irow </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">nrows</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">for</span><span class="token plain"> icol </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ncols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">try</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            feat </span><span class="token operator">=</span><span class="token plain"> features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow </span><span class="token operator">*</span><span class="token plain"> ncols </span><span class="token operator">+</span><span class="token plain"> icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            xx </span><span class="token operator">=</span><span class="token plain"> values</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">xx</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> yy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_yaxis</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ticks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">except</span><span class="token plain"> IndexError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">delaxes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">axes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">irow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">icol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="vw-style-feature-hashing">VW-style feature hashing<a href="#vw-style-feature-hashing" class="hash-link" aria-label="Direct link to VW-style feature hashing" title="Direct link to VW-style feature hashing">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_featurizer </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    inputCols</span><span class="token operator">=</span><span class="token plain">feature_cols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_train_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_test_data </span><span class="token operator">=</span><span class="token plain"> vw_featurizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training--prediction">Model training &amp; Prediction<a href="#model-training--prediction" class="hash-link" aria-label="Direct link to Model training &amp; Prediction" title="Direct link to Model training &amp; Prediction">​</a></h4><p>See <a href="https://github.com/vowpalWabbit/vowpal_wabbit/wiki/Command-Line-Arguments" target="_blank" rel="noopener noreferrer">VW wiki</a> for command line arguments.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">args </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -l 0.004 -q :: --power_t 0.3&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwr </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    passThroughArgs</span><span class="token operator">=</span><span class="token plain">args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># To reduce number of partitions (which will effect performance), use `vw_train_data.repartition(1)`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_model </span><span class="token operator">=</span><span class="token plain"> vwr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_train_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_predictions </span><span class="token operator">=</span><span class="token plain"> vw_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="compute-statistics--visualization">Compute Statistics &amp; Visualization<a href="#compute-statistics--visualization" class="hash-link" aria-label="Direct link to Compute Statistics &amp; Visualization" title="Direct link to Compute Statistics &amp; Visualization">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result </span><span class="token operator">=</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">insert</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_result</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmap </span><span class="token operator">=</span><span class="token plain"> get_cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YlOrRd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">target </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;target&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_preds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> axe </span><span class="token operator">=</span><span class="token plain"> plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">subplots</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">figsize</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">f</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tight_layout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">preds </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">vw_predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">err </span><span class="token operator">=</span><span class="token plain"> np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">absolute</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds </span><span class="token operator">-</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">norm </span><span class="token operator">=</span><span class="token plain"> Normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">clrs </span><span class="token operator">=</span><span class="token plain"> cmap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">np</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asarray</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">norm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">err</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">scatter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">preds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> target</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> s</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> c</span><span class="token operator">=</span><span class="token plain">clrs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> edgecolors</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> alpha</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">plt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">plot</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> line</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;#888888&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_xlabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Predicted values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_ylabel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Actual values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">axe</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_title</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Vowpal Wabbit&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="quantile-regression-for-drug-discovery-with-vowpalwabbitregressor">Quantile Regression for Drug Discovery with VowpalWabbitRegressor<a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="hash-link" aria-label="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor" title="Direct link to Quantile Regression for Drug Discovery with VowpalWabbitRegressor">​</a></h2><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/drug.png" width="800" class="img_ev3q"><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-2">Read dataset<a href="#read-dataset-2" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;libsvm&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/triazines.scale.svmlight&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token comment" style="color:rgb(136, 132, 111)"># print some basic info</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;records read: &quot;</span><span class="token plain"> </span><span class="token operator">+</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">count</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="split-dataset-into-train-and-test">Split dataset into train and test<a href="#split-dataset-into-train-and-test" class="hash-link" aria-label="Direct link to Split dataset into train and test" title="Direct link to Split dataset into train and test">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> triazines</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.85</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-training-2">Model Training<a href="#model-training-2" class="hash-link" aria-label="Direct link to Model Training" title="Direct link to Model Training">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> VowpalWabbitRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    numPasses</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> passThroughArgs</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="model-prediction-2">Model Prediction<a href="#model-prediction-2" class="hash-link" aria-label="Direct link to Model Prediction" title="Direct link to Model Prediction">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoredData </span><span class="token operator">=</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;regression&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoredData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vw-contextual-bandit">VW Contextual Bandit<a href="#vw-contextual-bandit" class="hash-link" aria-label="Direct link to VW Contextual Bandit" title="Direct link to VW Contextual Bandit">​</a></h2><h4 class="anchor anchorWithStickyNavbar_LWe7" id="read-dataset-3">Read dataset<a href="#read-dataset-3" class="hash-link" aria-label="Direct link to Read dataset" title="Direct link to Read dataset">​</a></h4><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/vwcb_input.dsjson&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Note: Actions are all five TAction_x_topic columns.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c.GUser.favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;c._multi.TAction.topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_Action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_labelIndex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cast</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;_label_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">print</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schema: &quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">printSchema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Add pipeline to add featurizer, convert all feature columns into vector.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            inputCols</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_0_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_1_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_2_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_3_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TAction_4_topic_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tranformation_pipeline </span><span class="token operator">=</span><span class="token plain"> pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformed_data </span><span class="token operator">=</span><span class="token plain"> tranformation_pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Build VowpalWabbit Contextual Bandit model and compute performance statistics.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">estimator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_id_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAdditionalSharedFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_major_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_hobby_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GUser_favorite_character_feature&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosenAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> estimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformed_data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getPerformanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Contextual Bandits/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contextual Bandits</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/Vowpal Wabbit/Quickstart - Classification using SparkML Vectors/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Classification using SparkML Vectors</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#advantages-of-vowpalwabbit" class="table-of-contents__link toc-highlight">Advantages of VowpalWabbit</a></li><li><a href="#why-use-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Why use VowpalWabbit on Spark?</a></li><li><a href="#operation-modes" class="table-of-contents__link toc-highlight">Operation modes</a><ul><li><a href="#limitations-of-vowpalwabbit-on-spark" class="table-of-contents__link toc-highlight">Limitations of VowpalWabbit on Spark</a></li><li><a href="#vowpalwabbit-usage" class="table-of-contents__link toc-highlight">VowpalWabbit Usage:</a></li></ul></li><li><a href="#heart-disease-detection-with-vowpalwabbit-classifier" class="table-of-contents__link toc-highlight">Heart Disease Detection with VowpalWabbit Classifier</a></li><li><a href="#adult-census-with-vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">Adult Census with VowpalWabbitClassifier</a></li><li><a href="#california-house-price-prediction-with-vowpalwabbitregressor---quantile-regression" class="table-of-contents__link toc-highlight">California house price prediction with VowpalWabbitRegressor - Quantile Regression</a></li><li><a href="#quantile-regression-for-drug-discovery-with-vowpalwabbitregressor" class="table-of-contents__link toc-highlight">Quantile Regression for Drug Discovery with VowpalWabbitRegressor</a></li><li><a href="#vw-contextual-bandit" class="table-of-contents__link toc-highlight">VW Contextual Bandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Get Started/Create a Spark Cluster/index.html b/docs/next/Get Started/Create a Spark Cluster/index.html
index b39d77cc67..e53852b31a 100644
--- a/docs/next/Get Started/Create a Spark Cluster/index.html	
+++ b/docs/next/Get Started/Create a Spark Cluster/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Create a Spark Cluster/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Create a Spark Cluster/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Create a Spark Cluster/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Create a Spark Cluster/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Create a Spark Cluster/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Create a Spark Cluster/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Create a Spark Cluster/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Create a Spark Cluster</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up your computing platform for SynapseML</h1><p>SynapseML is preinstalled on Microsoft Fabric and Synapse Analytics. Follow the instructions to get started with these platforms.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="microsoft-fabric">Microsoft Fabric<a href="#microsoft-fabric" class="hash-link" aria-label="Direct link to Microsoft Fabric" title="Direct link to Microsoft Fabric">​</a></h2><p><a href="https://www.microsoft.com/microsoft-fabric/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a> is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place.</p><p>SynapseML is preinstalled on Fabric, and this guide will walk you through getting access to fabric.</p><ul><li><a href="https://learn.microsoft.com/fabric/enterprise/licenses" target="_blank" rel="noopener noreferrer">Get a Microsoft Fabric license</a> or sign-up for a free <a href="https://learn.microsoft.com/fabric/get-started/fabric-trial" target="_blank" rel="noopener noreferrer">Microsoft Fabric (Preview) trial</a>.</li><li>Sign in to <a href="https://fabric.microsoft.com/" target="_blank" rel="noopener noreferrer">Microsoft Fabric</a></li><li>Go to the Data Science experience.</li><li><a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#create-notebooks" target="_blank" rel="noopener noreferrer">Create a new notebook</a> or attach your notebook to a lakehouse. On the left side, select <strong>Add</strong> to add an existing lakehouse or <a href="https://learn.microsoft.com/en-us/fabric/data-engineering/how-to-use-notebook#connect-lakehouses-and-notebooks" target="_blank" rel="noopener noreferrer">create a lakehouse</a>.</li></ul><p>SynapseML is preinstalled on Fabric, but if you want to use another version of SynapseML, follow <a href="https://learn.microsoft.com/en-us/fabric/data-science/install-synapseml" target="_blank" rel="noopener noreferrer">this guide on updating SynapseML</a>.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="synapse-analytics">Synapse Analytics<a href="#synapse-analytics" class="hash-link" aria-label="Direct link to Synapse Analytics" title="Direct link to Synapse Analytics">​</a></h2><p><a href="https://azure.microsoft.com/products/synapse-analytics" target="_blank" rel="noopener noreferrer">Azure Synapse Analytics</a> is an enterprise analytics service that accelerates time to insight across data warehouses and big data systems.</p><p>SynapseML is preinstalled on Synapse Analytics. To start with Synapse Analytics, you need:</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-create-workspace" target="_blank" rel="noopener noreferrer">Create a Synapse workspace and launch Synapse studio</a></li><li><a href="https://docs.microsoft.com/en-us/azure/synapse-analytics/get-started-analyze-spark#create-a-serverless-apache-spark-pool" target="_blank" rel="noopener noreferrer">Create a serverless Apache Spark pool</a></li><li>Once Synapse Studio has launched, select <strong>Develop</strong>. Then, select the <strong>&quot;+&quot;</strong> icon to add a new resource. From there, select <strong>Notebook</strong>. A new notebook is created and opened. Alternatively, you can select <strong>Import</strong> to upload your notebook.</li></ul><p>SynapseML is preinstalled on Azure Synapse Analytics, but if you want to use another version of SynapseML, follow <a href="/SynapseML/docs/next/Get Started/Install SynapseML/">this guide on updating SynapseML</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">What is SynapseML?</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install SynapseML</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse-analytics" class="table-of-contents__link toc-highlight">Synapse Analytics</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Get Started/Install SynapseML/index.html b/docs/next/Get Started/Install SynapseML/index.html
index 81d99a6066..581b9d51d8 100644
--- a/docs/next/Get Started/Install SynapseML/index.html	
+++ b/docs/next/Get Started/Install SynapseML/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 If you encounter issues, reach out to our support email!</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="r-beta">R (Beta)<a href="#r-beta" class="hash-link" aria-label="Direct link to R (Beta)" title="Direct link to R (Beta)">​</a></h2><p>To try out SynapseML using the R autogenerated wrappers, <a href="/SynapseML/docs/next/Reference/R Setup/">see our
 instructions</a>.  Note: This feature is still under development
 and some necessary custom wrappers may be missing.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Create a Spark Cluster</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Set up Cognitive Services</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#microsoft-fabric" class="table-of-contents__link toc-highlight">Microsoft Fabric</a></li><li><a href="#synapse" class="table-of-contents__link toc-highlight">Synapse</a></li><li><a href="#python" class="table-of-contents__link toc-highlight">Python</a></li><li><a href="#sbt" class="table-of-contents__link toc-highlight">SBT</a></li><li><a href="#spark-package" class="table-of-contents__link toc-highlight">Spark package</a></li><li><a href="#databricks" class="table-of-contents__link toc-highlight">Databricks</a></li><li><a href="#apache-livy-and-hdinsight" class="table-of-contents__link toc-highlight">Apache Livy and HDInsight</a></li><li><a href="#docker" class="table-of-contents__link toc-highlight">Docker</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from source</a></li><li><a href="#r-beta" class="table-of-contents__link toc-highlight">R (Beta)</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Get Started/Quickstart - Your First Models/index.html b/docs/next/Get Started/Quickstart - Your First Models/index.html
index 165499a087..36ae4647ff 100644
--- a/docs/next/Get Started/Quickstart - Your First Models/index.html	
+++ b/docs/next/Get Started/Quickstart - Your First Models/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Quickstart - Your First Models/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Quickstart - Your First Models/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Quickstart - Your First Models/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Quickstart - Your First Models/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Quickstart - Your First Models/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Quickstart - Your First Models</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Build your first SynapseML models</h1><p>This tutorial provides a brief introduction to SynapseML. In particular, we use SynapseML to create two different pipelines for sentiment analysis. The first pipeline combines a text featurization stage with LightGBM regression to predict ratings based on review text from a dataset containing book reviews from Amazon. The second pipeline shows how to use prebuilt models through the Azure AI Services to solve this problem without training data.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="load-a-dataset">Load a dataset<a href="#load-a-dataset" class="hash-link" aria-label="Direct link to Load a dataset" title="Direct link to Load a dataset">​</a></h2><p>Load your dataset and split it into train and test sets.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> test </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/BookReviewsFromAmazon10K.parquet&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">randomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="create-the-training-pipeline">Create the training pipeline<a href="#create-the-training-pipeline" class="hash-link" aria-label="Direct link to Create the training pipeline" title="Direct link to Create the training pipeline">​</a></h2><p>Create a pipeline that featurizes data using <code>TextFeaturizer</code> from the <code>synapse.ml.featurize.text</code> library and derives a rating using the <code>LightGBMRegressor</code> function.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml </span><span class="token keyword">import</span><span class="token plain"> Pipeline</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> TextFeaturizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> LightGBMRegressor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> Pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featuresCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="predict-the-output-of-the-test-data">Predict the output of the test data<a href="#predict-the-output-of-the-test-data" class="hash-link" aria-label="Direct link to Predict the output of the test data" title="Direct link to Predict the output of the test data">​</a></h2><p>Call the <code>transform</code> function on the model to predict and display the output of the test data as a dataframe.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="use-azure-ai-services-to-transform-data-in-one-step">Use Azure AI services to transform data in one step<a href="#use-azure-ai-services-to-transform-data-in-one-step" class="hash-link" aria-label="Direct link to Use Azure AI services to transform data in one step" title="Direct link to Use Azure AI services to transform data in one step">​</a></h2><p>Alternatively, for these kinds of tasks that have a prebuilt solution, you can use SynapseML&#x27;s integration with Azure AI services to transform your data in one step.</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    textCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    kind</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SentimentAnalysis&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># Replace the call to find_secret with your key as a python string.</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Set up Cognitive Services</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Overview</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#load-a-dataset" class="table-of-contents__link toc-highlight">Load a dataset</a></li><li><a href="#create-the-training-pipeline" class="table-of-contents__link toc-highlight">Create the training pipeline</a></li><li><a href="#predict-the-output-of-the-test-data" class="table-of-contents__link toc-highlight">Predict the output of the test data</a></li><li><a href="#use-azure-ai-services-to-transform-data-in-one-step" class="table-of-contents__link toc-highlight">Use Azure AI services to transform data in one step</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Get Started/Set up Cognitive Services/index.html b/docs/next/Get Started/Set up Cognitive Services/index.html
index b45e3e3c7d..a9d33ae7c4 100644
--- a/docs/next/Get Started/Set up Cognitive Services/index.html	
+++ b/docs/next/Get Started/Set up Cognitive Services/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Get Started/Set up Cognitive Services/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Get Started/Set up Cognitive Services/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Get Started/Set up Cognitive Services/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Get Started/Set up Cognitive Services/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Get Started/Set up Cognitive Services/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Get Started/Set up Cognitive Services/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Create a Spark Cluster</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Install SynapseML/">Install SynapseML</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Get Started/Set up Cognitive Services/">Set up Cognitive Services</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/">Quickstart - Your First Models</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Get Started/Set up Cognitive Services/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Get Started</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Set up Cognitive Services</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Setting up Azure AI Services and Azure OpenAI resources for SynapseML</h1><p>In order to use SynapseML&#x27;s OpenAI or Azure AI Services features, specific Azure resources are required. This documentation walks you through the process of setting up these resources and acquiring the necessary credentials.</p><p>First, create an Azure subscription to create resources.</p><ul><li>A valid Azure subscription - <a href="https://azure.microsoft.com/free/cognitive-services/" target="_blank" rel="noopener noreferrer">Create one for free</a>.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-openai">Azure OpenAI<a href="#azure-openai" class="hash-link" aria-label="Direct link to Azure OpenAI" title="Direct link to Azure OpenAI">​</a></h2><p>The <a href="https://azure.microsoft.com/products/cognitive-services/openai-service/" target="_blank" rel="noopener noreferrer">Azure OpenAI service</a> can be used to solve a large number of natural language tasks through prompting the completion API. To make it easier to scale your prompting workflows from a few examples to large datasets of examples, we have integrated the Azure OpenAI service with the distributed machine learning library SynapseML. This integration makes it easy to use the Apache Spark distributed computing framework to process millions of prompts with the OpenAI service.</p><p>To set up your Azure OpenAI Resource for SynapseML usage you need to: </p><ul><li><a href="https://aka.ms/oai/access" target="_blank" rel="noopener noreferrer">Apply for access to Azure OpenAI</a> if you do not already have access. </li><li><a href="https://docs.microsoft.com/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal#create-a-resource" target="_blank" rel="noopener noreferrer">Create an Azure OpenAI resource</a> </li><li>Get your Azure OpenAI resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h2><p>To set up <a href="https://azure.microsoft.com/en-us/products/ai-services" target="_blank" rel="noopener noreferrer">Azure AI Services</a> for use with SynapseML you first need to:</p><ul><li><a href="https://learn.microsoft.com/azure/role-based-access-control/role-assignments-steps" target="_blank" rel="noopener noreferrer">Assign yourself the Azure AI Services Contributor role</a> to agree to the responsible AI terms and create a resource. </li><li><a href="https://ms.portal.azure.com/#create/Microsoft.CognitiveServicesAllInOne" target="_blank" rel="noopener noreferrer">Create Azure AI service (Decision, Language, Speech, Vision) resource</a>. You can follow the steps at <a href="https://learn.microsoft.com/en-us/azure/ai-services/multi-service-resource?tabs=windows&amp;pivots=azportal#create-a-new-azure-cognitive-services-resource" target="_blank" rel="noopener noreferrer">Create a multi-service resource for Azure AI services</a>. </li><li>Get your Azure AI Services resource&#x27;s key. After your resource is successfully deployed, select <strong>Next Steps</strong> &gt; <strong>Go to resource</strong>. Once at the resource, you can get the key from <strong>Resource Management</strong> &gt; <strong>Keys and Endpoint</strong>. Copy the key and paste it into the notebook. Store keys securely and do not share them. </li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret">Manage secrets with Azure Key Vault and access the secrets with find_secret<a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="hash-link" aria-label="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret" title="Direct link to Manage secrets with Azure Key Vault and access the secrets with find_secret">​</a></h2><p>After you create an Azure AI resource, you will obtain a resource key. You can use this resource key directly in our notebooks as a string, but we recommend to manage secrets with Azure Key Vault. Azure Key Vault is a cloud-based service that allows you to store and manage cryptographic keys, certificates, and secrets used by cloud applications and services.</p><p>You can skip the following content if you want to use a secret string instead of Azure Key Vault. This is not recommended for production workloads.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-key-vault">Create Azure Key Vault<a href="#create-azure-key-vault" class="hash-link" aria-label="Direct link to Create Azure Key Vault" title="Direct link to Create Azure Key Vault">​</a></h3><p>Refer to <a href="https://learn.microsoft.com/azure/key-vault/general/quick-create-portal" target="_blank" rel="noopener noreferrer">this article</a> to create a key vault using Azure Portal</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="save-secret-to-azure-key-vault">Save secret to Azure Key Vault<a href="#save-secret-to-azure-key-vault" class="hash-link" aria-label="Direct link to Save secret to Azure Key Vault" title="Direct link to Save secret to Azure Key Vault">​</a></h3><ul><li>Go to Access control (IAM) and assign <strong>Key Vault Administrator Role</strong> to yourself.</li><li>Go to Secrets and choose <strong>+ Generate/Import</strong>, create a key with the secret value obtained from Azure AI service.</li><li>Choose <strong>Create</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-microsoft-fabric--power-bi">Use find_secret on Microsoft Fabric / Power BI<a href="#use-find_secret-on-microsoft-fabric--power-bi" class="hash-link" aria-label="Direct link to Use find_secret on Microsoft Fabric / Power BI" title="Direct link to Use find_secret on Microsoft Fabric / Power BI">​</a></h3><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Granted the Fabric or Power BI Service Account Get permissions in the Azure Key Vault.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-azure-databricks">Use find_secret on Azure Databricks<a href="#use-find_secret-on-azure-databricks" class="hash-link" aria-label="Direct link to Use find_secret on Azure Databricks" title="Direct link to Use find_secret on Azure Databricks">​</a></h3><ul><li>In the Azure Portal, find and select the Azure Key Vault Instance. Select the <strong>Access configuration</strong> tab under Settings. Set Permission model to Vault access policy.</li><li>On Databricks, go to <code>https://&lt;databricks-instance&gt;#secrets/createScope</code>. This URL is case sensitive.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_create_secret_scope.png" width="600" class="img_ev3q"><ul><li>Enter the name of the secret scope and choose desired Manage Principal.</li><li>Go to your Azure Key Vault -&gt; Properties and find the <strong>DNS Name</strong> (Vault URI on Azure Key Vault) and <strong>Resource ID</strong>. Enter the DNS Name and Resource ID on Databricks createScope page.</li></ul><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_find_resource_id.png" width="600" class="img_ev3q"><ul><li>Click the <strong>Create</strong> Button.</li></ul><p>Refer to <a href="https://learn.microsoft.com/azure/databricks/security/secrets/secret-scopes#--create-an-azure-key-vault-backed-secret-scope" target="_blank" rel="noopener noreferrer">this article</a> for more details.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="use-find_secret-on-synapse">Use find_secret on Synapse<a href="#use-find_secret-on-synapse" class="hash-link" aria-label="Direct link to Use find_secret on Synapse" title="Direct link to Use find_secret on Synapse">​</a></h3><ul><li>Open the Synapse Studio and go to the Manage tab.</li><li>Under External connections, select Linked services.</li><li>To add a linked service, select New.</li><li>Select the Azure Key Vault tile from the list and select Continue.</li><li>Enter a linked service name and choose the key vault you want to connect to.</li><li>Select Create</li><li>Click Publish</li></ul><ul><li>Go to Azure Key Vault, Select Access control (IAM), Select <strong>+ Add</strong>, Add role assignment.</li><li>Choose <strong>Key Vault Administrator</strong> in Role blade, select Next.</li><li>In Members blade, choose Assign access to <strong>Managed identity</strong>. Select members, choose the subscription your Synapse Workspace in. For Managed identity, select Synapse workspace, choose your workspace.</li><li>Select <strong>Review + assign</strong>.</li></ul><h3 class="anchor anchorWithStickyNavbar_LWe7" id="quick-test">Quick Test<a href="#quick-test" class="hash-link" aria-label="Direct link to Quick Test" title="Direct link to Quick Test">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">language </span><span class="token keyword">import</span><span class="token plain"> AnalyzeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">core</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">platform </span><span class="token keyword">import</span><span class="token plain"> find_secret</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_key </span><span class="token operator">=</span><span class="token plain"> find_secret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    secret_name</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ai-services-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> keyvault</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-build-keys&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own key vault name and api name</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai_service_location </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token plain">  </span><span class="token comment" style="color:rgb(136, 132, 111)"># use your own AI service location</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    data</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Dr. Smith has a very modern medical office, and she has great staff.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    schema</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity_recognition </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    AnalyzeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKind</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EntityRecognition&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_location</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ai_service_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df_results </span><span class="token operator">=</span><span class="token plain"> entity_recognition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_results</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;entities.documents.entities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Your result should looks like</p><table><thead><tr><th>language</th><th>text</th><th>entities</th></tr></thead><tbody><tr><td>en</td><td>Dr. Smith has a very modern medical office, and she has great staff.</td><td>[{&quot;category&quot;: &quot;Person&quot;, &quot;confidenceScore&quot;: 0.98, &quot;length&quot;: 5, &quot;offset&quot;: 4, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;Smith&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 0.79, &quot;length&quot;: 14, &quot;offset&quot;: 28, &quot;subcategory&quot;: &quot;Structural&quot;, &quot;text&quot;: &quot;medical office&quot;}, {&quot;category&quot;: &quot;PersonType&quot;, &quot;confidenceScore&quot;: 0.85, &quot;length&quot;: 5, &quot;offset&quot;: 62, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;staff&quot;}]</td></tr><tr><td>en</td><td>I had a wonderful trip to Seattle last week.</td><td>[{&quot;category&quot;: &quot;Event&quot;, &quot;confidenceScore&quot;: 0.74, &quot;length&quot;: 4, &quot;offset&quot;: 18, &quot;subcategory&quot;: null, &quot;text&quot;: &quot;trip&quot;}, {&quot;category&quot;: &quot;Location&quot;, &quot;confidenceScore&quot;: 1, &quot;length&quot;: 7, &quot;offset&quot;: 26, &quot;subcategory&quot;: &quot;GPE&quot;, &quot;text&quot;: &quot;Seattle&quot;}, {&quot;category&quot;: &quot;DateTime&quot;, &quot;confidenceScore&quot;: 0.8, &quot;length&quot;: 9, &quot;offset&quot;: 34, &quot;subcategory&quot;: &quot;DateRange&quot;, &quot;text&quot;: &quot;last week&quot;}]</td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Get Started/Install SynapseML/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install SynapseML</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Get Started/Quickstart - Your First Models/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Quickstart - Your First Models</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#azure-openai" class="table-of-contents__link toc-highlight">Azure OpenAI</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li><li><a href="#manage-secrets-with-azure-key-vault-and-access-the-secrets-with-find_secret" class="table-of-contents__link toc-highlight">Manage secrets with Azure Key Vault and access the secrets with find_secret</a><ul><li><a href="#create-azure-key-vault" class="table-of-contents__link toc-highlight">Create Azure Key Vault</a></li><li><a href="#save-secret-to-azure-key-vault" class="table-of-contents__link toc-highlight">Save secret to Azure Key Vault</a></li><li><a href="#use-find_secret-on-microsoft-fabric--power-bi" class="table-of-contents__link toc-highlight">Use find_secret on Microsoft Fabric / Power BI</a></li><li><a href="#use-find_secret-on-azure-databricks" class="table-of-contents__link toc-highlight">Use find_secret on Azure Databricks</a></li><li><a href="#use-find_secret-on-synapse" class="table-of-contents__link toc-highlight">Use find_secret on Synapse</a></li><li><a href="#quick-test" class="table-of-contents__link toc-highlight">Quick Test</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Overview/index.html b/docs/next/Overview/index.html
index feee862df0..109f55739d 100644
--- a/docs/next/Overview/index.html
+++ b/docs/next/Overview/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a aria-current="page" class="navbar__link active" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">What is SynapseML?</span><meta itemprop="position" content="1"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>What is SynapseML?</h1><p>SynapseML (previously known as MMLSpark), is an open-source library that simplifies the creation of massively scalable machine learning (ML) pipelines. SynapseML provides simple, composable, and distributed APIs for a wide variety of different machine learning tasks such as text analytics, vision, anomaly detection, and many others. SynapseML is built on the <a href="https://spark.apache.org/" target="_blank" rel="noopener noreferrer">Apache Spark distributed computing framework</a> and shares the same API as the <a href="https://spark.apache.org/mllib/" target="_blank" rel="noopener noreferrer">SparkML/MLLib library</a>, allowing you to seamlessly embed SynapseML models into existing Apache Spark workflows.</p><p>With SynapseML, you can build scalable and intelligent systems to solve challenges in domains such as anomaly detection, computer vision, deep learning, text analytics, and others. SynapseML can train and evaluate models on single-node, multi-node, and elastically resizable clusters of computers. This lets you scale your work without wasting resources. SynapseML is usable across Python, R, Scala, Java, and .NET. Furthermore, its API abstracts over a wide variety of databases, file systems, and cloud data stores to simplify experiments no matter where data is located.</p><p>SynapseML requires Scala 2.12, Spark 3.2+, and Python 3.8+.</p><a class="button button--lg button--outline button--block button--primary" href="/SynapseML/docs/Get Started/Install SynapseML/">Get Started</a><h2 class="anchor anchorWithStickyNavbar_LWe7" id="papers">Papers<a href="#papers" class="hash-link" aria-label="Direct link to Papers" title="Direct link to Papers">​</a></h2><ul><li><p><a href="https://arxiv.org/abs/2009.08044" target="_blank" rel="noopener noreferrer">Large Scale Intelligent Microservices</a></p></li><li><p><a href="https://arxiv.org/abs/2007.07177" target="_blank" rel="noopener noreferrer">Conditional Image Retrieval</a></p></li><li><p><a href="https://arxiv.org/abs/1810.08744" target="_blank" rel="noopener noreferrer">SynapseML: Unifying Machine Learning Ecosystems at Massive Scales</a></p></li><li><p><a href="https://arxiv.org/abs/1804.04031" target="_blank" rel="noopener noreferrer">Flexible and Scalable Deep Learning with MMLSpark</a></p></li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Create a Spark Cluster</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#papers" class="table-of-contents__link toc-highlight">Papers</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/estimators/estimators_causal/index.html b/docs/next/Quick Examples/estimators/estimators_causal/index.html
index 563486a052..7845d261dc 100644
--- a/docs/next/Quick Examples/estimators/estimators_causal/index.html	
+++ b/docs/next/Quick Examples/estimators/estimators_causal/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_causal/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_causal/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_causal/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_causal/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_causal/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/estimators/estimators_causal/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_causal/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Causal Inference</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="doublemlestimator">DoubleMLEstimator<a href="#doublemlestimator" class="hash-link" aria-label="Direct link to DoubleMLEstimator" title="Direct link to DoubleMLEstimator">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">causal </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> IntegerType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">schema </span><span class="token operator">=</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> BooleanType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DoubleType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.38</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.98</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.15</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.65</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">8.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.45</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.57</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.97</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">7.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> schema</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DoubleMLEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Treatment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTreatmentModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Outcome&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutcomeModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxIter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel </span><span class="token operator">=</span><span class="token plain"> dml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getAvgTreatmentEffect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getConfidenceInterval</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.causal._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, false, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, true, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (false, true, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (true, false, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Treatment&quot;, &quot;Outcome&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dml = (new DoubleMLEstimator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentCol(&quot;Treatment&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTreatmentModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeCol(&quot;Outcome&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutcomeModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxIter(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmlModel = dml.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getAvgTreatmentEffect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmlModel.getConfidenceInterval</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.causal.html#module-synapse.ml.causal.DoubleMLEstimator">DoubleMLEstimator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.html">DoubleMLEstimator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/causal/DoubleMLEstimator.scala">DoubleMLEstimator</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#doublemlestimator" class="table-of-contents__link toc-highlight">DoubleMLEstimator</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/estimators/estimators_cognitive/index.html b/docs/next/Quick Examples/estimators/estimators_cognitive/index.html
index 9d705cb48e..2bd5220e3e 100644
--- a/docs/next/Quick Examples/estimators/estimators_cognitive/index.html	
+++ b/docs/next/Quick Examples/estimators/estimators_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/estimators/estimators_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="simplefitmultivariateanomaly">SimpleFitMultivariateAnomaly<a href="#simplefitmultivariateanomaly" class="hash-link" aria-label="Direct link to SimpleFitMultivariateAnomaly" title="Direct link to SimpleFitMultivariateAnomaly">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">startTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-01T00:00:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">endTime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2021-01-03T01:59:00Z&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timestampColumn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">inputColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;feature2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">intermediateSaveDir </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleFitMultivariateAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStartTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">startTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEndTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">endTime</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIntermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">intermediateSaveDir</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimestampCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">timestampColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSlidingWindow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># uncomment below for fitting your own dataframe</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># model = simpleFitMultivariateAnomaly.fit(df)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.FitMultivariateAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val startTime: String = &quot;2021-01-01T00:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val endTime: String = &quot;2021-01-02T12:00:00Z&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timestampColumn: String = &quot;timestamp&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val inputColumns: Array[String] = Array(&quot;feature0&quot;, &quot;feature1&quot;, &quot;feature2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val intermediateSaveDir: String = &quot;wasbs://madtest@anomalydetectiontest.blob.core.windows.net/intermediateData&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val simpleFitMultivariateAnomaly = (new SimpleFitMultivariateAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIntermediateSaveDir(intermediateSaveDir)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSlidingWindow(50))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (spark.read.format(&quot;csv&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .option(&quot;header&quot;, True)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/MAD/mad_example.csv&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = simpleFitMultivariateAnomaly.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val result = (model</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setStartTime(startTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEndTime(endTime)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTimestampCol(timestampColumn)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setInputCols(inputColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">result.show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">simpleFitMultivariateAnomaly.cleanUpIntermediateData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model.cleanUpIntermediateData()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleFitMultivariateAnomaly">SimpleFitMultivariateAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleFitMultivariateAnomaly.html">SimpleFitMultivariateAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/MultivariateAnomalyDetection.scala">SimpleFitMultivariateAnomaly</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#simplefitmultivariateanomaly" class="table-of-contents__link toc-highlight">SimpleFitMultivariateAnomaly</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/estimators/estimators_core/index.html b/docs/next/Quick Examples/estimators/estimators_core/index.html
index 43a3681dba..9f3108c166 100644
--- a/docs/next/Quick Examples/estimators/estimators_core/index.html	
+++ b/docs/next/Quick Examples/estimators/estimators_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/estimators/estimators_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automl">AutoML<a href="#automl" class="hash-link" aria-label="Direct link to AutoML" title="Direct link to AutoML">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findbestmodel">FindBestModel<a href="#findbestmodel" class="hash-link" aria-label="Direct link to FindBestModel" title="Direct link to FindBestModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> RandomForestClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># mocking models</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomForestClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">32</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInfoGain</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinInstancesPerNode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubsamplingRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mlfeatures&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> randomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findBestModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> findBestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.Transformer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// mocking models</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomForestClassifier = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBins(32)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxDepth(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInfoGain(0.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMinInstancesPerNode(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setNumTrees(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSubsamplingRate(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setSeed(0L))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;mlfeatures&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = randomForestClassifier.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findBestModel = (new FindBestModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(Array(model.asInstanceOf[Transformer], model.asInstanceOf[Transformer]))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setEvaluationMetric(&quot;accuracy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bestModel = findBestModel.fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.FindBestModel">FindBestModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.html">FindBestModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/FindBestModel.scala">FindBestModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tunehyperparameters">TuneHyperparameters<a href="#tunehyperparameters" class="hash-link" aria-label="Direct link to TuneHyperparameters" title="Direct link to TuneHyperparameters">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">automl </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> GBTClassifier</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Clump_Thickness&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Uniformity_of_Cell_Shape&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Marginal_Adhesion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Single_Epithelial_Cell_Size&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bare_Nuclei&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bland_Chromatin&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Normal_Nucleoli&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mitoses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">logReg </span><span class="token operator">=</span><span class="token plain"> LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randForest </span><span class="token operator">=</span><span class="token plain"> RandomForestClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gbt </span><span class="token operator">=</span><span class="token plain"> GBTClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">smlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mmlmodels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token operator">=</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> model </span><span class="token keyword">in</span><span class="token plain"> smlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramBuilder </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HyperparamBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> logReg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">numTrees</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> randForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxBins</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> RangeHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addHyperparam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> gbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">maxDepth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> DiscreteHyperParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">searchSpace </span><span class="token operator">=</span><span class="token plain"> paramBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># The search space is a list of params to tuples of estimator and hyperparam</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">randomSpace </span><span class="token operator">=</span><span class="token plain"> RandomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">searchSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bestModel </span><span class="token operator">=</span><span class="token plain"> TuneHyperparameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> models</span><span class="token operator">=</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> numFolds</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              numRuns</span><span class="token operator">=</span><span class="token builtin" style="color:rgb(174, 129, 255)">len</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mmlmodels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> parallelism</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              paramSpace</span><span class="token operator">=</span><span class="token plain">randomSpace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">space</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> seed</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.automl._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logReg = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randForest = new RandomForestClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gbt = new GBTClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val smlmodels = Seq(logReg, randForest, gbt)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mmlmodels = smlmodels.map(model =&gt; new TrainClassifier().setModel(model).setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramBuilder = new HyperparamBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(logReg.regParam, new DoubleRangeHyperParam(0.1, 0.3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.numTrees, new DiscreteHyperParam(List(5,10)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(randForest.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .addHyperparam(gbt.maxBins, new IntRangeHyperParam(8,16))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">.addHyperparam(gbt.maxDepth, new DiscreteHyperParam(List(3,5)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val searchSpace = paramBuilder.build()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randomSpace = new RandomSpace(searchSpace)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 1, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 1, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 1, 1, 1, 2, 1.0, 2, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 1, 2, 3, 1, 2, 1.0, 3, 1, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (0, 3, 1, 1, 1, 2, 1.0, 3, 1, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;Label&quot;, &quot;Clump_Thickness&quot;, &quot;Uniformity_of_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Uniformity_of_Cell_Shape&quot;, &quot;Marginal_Adhesion&quot;, &quot;Single_Epithelial_Cell_Size&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Bare_Nuclei&quot;, &quot;Bland_Chromatin&quot;, &quot;Normal_Nucleoli&quot;, &quot;Mitoses&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tuneHyperparameters = new TuneHyperparameters().setEvaluationMetric(&quot;accuracy&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModels(mmlmodels.toArray).setNumFolds(2).setNumRuns(mmlmodels.length * 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setParallelism(1).setParamSpace(randomSpace).setSeed(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tuneHyperparameters.fit(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.automl.html#module-synapse.ml.automl.TuneHyperparameters">TuneHyperparameters</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.html">TuneHyperparameters</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/automl/TuneHyperparameters.scala">TuneHyperparameters</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleanmissingdata">CleanMissingData<a href="#cleanmissingdata" class="hash-link" aria-label="Direct link to CleanMissingData" title="Direct link to CleanMissingData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">    </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col5&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cmd </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CleanMissingData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCleaningMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Mean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.lang.{Boolean =&gt; JBoolean, Double =&gt; JDouble, Integer =&gt; JInt}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createMockDataset: DataFrame = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Seq[(JInt, JInt, JDouble, JDouble, JInt)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    2,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    3,    0.40, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    4,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    5,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    1,    0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (null, null, null, null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    null, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    2,    0.40, 0.50, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0,    3,    null, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1,    4,    0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;, &quot;col5&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = createMockDataset</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cmd = (new CleanMissingData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCols(dataset.columns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setCleaningMode(&quot;Mean&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CleanMissingData">CleanMissingData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.html">CleanMissingData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CleanMissingData.scala">CleanMissingData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="countselector">CountSelector<a href="#countselector" class="hash-link" aria-label="Direct link to CountSelector" title="Direct link to CountSelector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sparse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs </span><span class="token operator">=</span><span class="token plain"> CountSelector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (Vectors.sparse(3, Seq((0, 1.0), (2, 2.0))), Vectors.dense(1.0, 0.1, 0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;col1&quot;, &quot;col2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cs = (new CountSelector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setInputCol(&quot;col1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cs.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.CountSelector">CountSelector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.html">CountSelector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/CountSelector.scala">CountSelector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-1">Featurize<a href="#featurize-1" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon are everywhere&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the woods&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the water&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;they are in the fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pokemon - gotta catch em all&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;testColumn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, &quot;pokemon are everywhere&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, &quot;they are in the woods&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, &quot;they are in the water&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, &quot;they are in the fields&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, &quot;pokemon - gotta catch em all&quot;)).toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featureColumns = dataset.columns.filter(_ != &quot;Label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val feat = (new Featurize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;testColumn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(featureColumns)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOneHotEncodeCategoricals(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">feat.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.Featurize">Featurize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.html">Featurize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/Featurize.scala">Featurize</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="valueindexer">ValueIndexer<a href="#valueindexer" class="hash-link" aria-label="Direct link to ValueIndexer" title="Direct link to ValueIndexer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vi = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vi.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.ValueIndexer">ValueIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.html">ValueIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/ValueIndexer.scala">ValueIndexer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textfeaturizer">TextFeaturizer<a href="#textfeaturizer" class="hash-link" aria-label="Direct link to TextFeaturizer" title="Direct link to TextFeaturizer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = Seq((0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            (3, &quot;&quot;)).toDF(&quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tfRaw = (new TextFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumFeatures(20))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tfRaw.fit(dfRaw).transform(dfRaw).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.TextFeaturizer">TextFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.html">TextFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/TextFeaturizer.scala">TextFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="isolation-forest">Isolation Forest<a href="#isolation-forest" class="hash-link" aria-label="Direct link to Isolation Forest" title="Direct link to Isolation Forest">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="isolationforest">IsolationForest<a href="#isolationforest" class="hash-link" aria-label="Direct link to IsolationForest" title="Direct link to IsolationForest">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">isolationforest </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isolationForest </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IsolationForest</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumEstimators</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBootstrap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">256</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;predictedLabel&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoreCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;outlierScore&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContamination</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setContaminationError</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.02</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.01</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRandomSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.isolationforest._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isolationForest = (new IsolationForest()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumEstimators(100)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBootstrap(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxSamples(256)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxFeatures(1.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;predictedLabel&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoreCol(&quot;outlierScore&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContamination(0.02)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setContaminationError(0.02 * 0.01)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRandomSeed(1))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.isolationforest.html#module-synapse.ml.isolationforest.IsolationForest">IsolationForest</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.html">IsolationForest</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/isolationforest/IsolationForest.scala">IsolationForest</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="nn">NN<a href="#nn" class="hash-link" aria-label="Direct link to NN" title="Direct link to NN">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="conditionalknn">ConditionalKNN<a href="#conditionalknn" class="hash-link" aria-label="Direct link to ConditionalKNN" title="Direct link to ConditionalKNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cknn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cknn = (new ConditionalKNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setOutputCol(&quot;matches&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setFeaturesCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.ConditionalKNN">ConditionalKNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.html">ConditionalKNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/ConditionalKNN.scala">ConditionalKNN</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="knn">KNN<a href="#knn" class="hash-link" aria-label="Direct link to KNN" title="Direct link to KNN">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">knn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;matches&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.nn._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val knn = (new KNN()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;matches&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.nn.html#module-synapse.ml.nn.KNN">KNN</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/nn/KNN.html">KNN</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/nn/KNN.scala">KNN</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="recommendation">Recommendation<a href="#recommendation" class="hash-link" aria-label="Direct link to Recommendation" title="Direct link to Recommendation">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit<a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="hash-link" aria-label="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit" title="Direct link to RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">tuning </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transformedDf </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">als </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ALS</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumUserBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumItemBlocks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeed</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">evaluator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNItems</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">paramGrid </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ParamGridBuilder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">addGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">build</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingTrainValidationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">als</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEstimatorParamMaps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">paramGrid</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTrainRatio</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformedDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.recommendation.ALS</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.tuning._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transformedDf = (recommendationIndexer.fit(ratings)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(ratings).cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val als = (new ALS()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumUserBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumItemBlocks(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSeed(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val evaluator = (new RankingEvaluator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNItems(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setK(evaluator.getK)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRecommender(als))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(transformedDf).transform(transformedDf).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val paramGrid = (new ParamGridBuilder()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .addGrid(als.regParam, Array(1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .build())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tvRecommendationSplit = (new RankingTrainValidationSplit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimator(als)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluator(evaluator)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEstimatorParamMaps(paramGrid)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTrainRatio(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(recommendationIndexer.getUserOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(recommendationIndexer.getItemOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tvRecommendationSplit.fit(transformedDf).transform(transformedDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RecommendationIndexer">RecommendationIndexer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.html">RecommendationIndexer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RecommendationIndexer.scala">RecommendationIndexer</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingEvaluator">RankingEvaluator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.html">RankingEvaluator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingEvaluator.scala">RankingEvaluator</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingAdapter">RankingAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.html">RankingAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingAdapter.scala">RankingAdapter</a></td></tr></tbody></table><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.RankingTrainValidationSplit">RankingTrainValidationSplit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.html">RankingTrainValidationSplit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/RankingTrainValidationSplit.scala">RankingTrainValidationSplit</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sar">SAR<a href="#sar" class="hash-link" aria-label="Direct link to SAR" title="Direct link to SAR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">recommendation </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ratings </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;11&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;22&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 04&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 09&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;33&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 01&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 02&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 03&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 05&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 06&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 07&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 08&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;44&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Movie 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dropDuplicates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">recommendationIndexer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemIDOrg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">algo </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SAR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUserCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;customerID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setItemCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;itemID&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRatingCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rating&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTimeCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSupportThreshold</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSimilarityFunction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;jacccard&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActivityTimeFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RankingAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRecommender</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">algo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">res1 </span><span class="token operator">=</span><span class="token plain"> recommendationIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ratings</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">cache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">res1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.recommendation._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ratings = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 01&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 07&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 08&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;11&quot;, &quot;Movie 09&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 06&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;22&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 04&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 09&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;33&quot;, &quot;Movie 10&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 01&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 02&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 03&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 05&quot;, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 06&quot;, 4),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 07&quot;, 5),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 08&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;44&quot;, &quot;Movie 10&quot;, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;customerIDOrg&quot;, &quot;itemIDOrg&quot;, &quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .dropDuplicates()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val recommendationIndexer = (new RecommendationIndexer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserInputCol(&quot;customerIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setUserOutputCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemInputCol(&quot;itemIDOrg&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setItemOutputCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setRatingCol(&quot;rating&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val algo = (new SAR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUserCol(&quot;customerID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setItemCol(&quot;itemID&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRatingCol(&quot;rating&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTimeCol(&quot;timestamp&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSupportThreshold(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setSimilarityFunction(&quot;jacccard&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setActivityTimeFormat(&quot;EEE MMM dd HH:mm:ss Z yyyy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val adapter = (new RankingAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setK(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRecommender(algo))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val res1 = recommendationIndexer.fit(ratings).transform(ratings).cache()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">adapter.fit(res1).transform(res1).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.recommendation.html#module-synapse.ml.recommendation.SAR">SAR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.html">SAR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/recommendation/SAR.scala">SAR</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="classbalancer">ClassBalancer<a href="#classbalancer" class="hash-link" aria-label="Direct link to ClassBalancer" title="Direct link to ClassBalancer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;index&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> ClassBalancer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1.0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, 2.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, 0.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, 1.0, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, 0.0, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, 2.0, &quot;&quot;)).toDF(&quot;index&quot;, &quot;label&quot;, &quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = new ClassBalancer().setInputCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.ClassBalancer">ClassBalancer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.html">ClassBalancer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/ClassBalancer.scala">ClassBalancer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multicolumnadapter">MultiColumnAdapter<a href="#multicolumnadapter" class="hash-link" aria-label="Direct link to MultiColumnAdapter" title="Direct link to MultiColumnAdapter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;This is a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;this is one too&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;could be a test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;maybe not&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stage1 </span><span class="token operator">=</span><span class="token plain"> Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiColumnAdapter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBaseStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stage1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;This is a test&quot;, &quot;this is one too&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;could be a test&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;foo&quot;, &quot;bar&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;foo&quot;, &quot;maybe not&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;words1&quot;, &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stage1 = new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mca = (new MultiColumnAdapter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setBaseStage(stage1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCols(Array[String](&quot;words1&quot;,  &quot;words2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCols(Array[String](&quot;output1&quot;, &quot;output2&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mca.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.MultiColumnAdapter">MultiColumnAdapter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.html">MultiColumnAdapter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MultiColumnAdapter.scala">MultiColumnAdapter</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timer">Timer<a href="#timer" class="hash-link" aria-label="Direct link to Timer" title="Direct link to Timer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">tok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df3 </span><span class="token operator">=</span><span class="token plain"> HashingTF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">idf </span><span class="token operator">=</span><span class="token plain"> IDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hash&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;idf&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer </span><span class="token operator">=</span><span class="token plain"> Timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">idf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;tokens&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new Timer().setStage(tok).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df3 = new HashingTF().setInputCol(&quot;tokens&quot;).setOutputCol(&quot;hash&quot;).transform(df2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val idf = new IDF().setInputCol(&quot;hash&quot;).setOutputCol(&quot;idf&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timer = new Timer().setStage(idf)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timer.fit(df3).transform(df3).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Timer">Timer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Timer.html">Timer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Timer.scala">Timer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainclassifier">TrainClassifier<a href="#trainclassifier" class="hash-link" aria-label="Direct link to TrainClassifier" title="Direct link to TrainClassifier">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">classification </span><span class="token keyword">import</span><span class="token plain"> LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LogisticRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 2, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 4, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 5, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 1, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 3, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0, 0.50, 0.60, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 2, 0.40, 0.50, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 3, 0.78, 0.99, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 4, 0.12, 0.34, 3))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;Label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val tc = (new TrainClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(new LogisticRegression())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;Label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">tc.fit(df).transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainClassifier">TrainClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.html">TrainClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainClassifier.scala">TrainClassifier</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="trainregressor">TrainRegressor<a href="#trainregressor" class="hash-link" aria-label="Direct link to TrainRegressor" title="Direct link to TrainRegressor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">regression </span><span class="token keyword">import</span><span class="token plain"> LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dataset </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.78</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.99</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linearRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LinearRegression</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRegParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setElasticNetParam</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TrainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linearRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dataset</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = (spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;col4&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val linearRegressor = (new LinearRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val trainRegressor = (new TrainRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setModel(linearRegressor)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">trainRegressor.fit(dataset).transform(dataset).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.TrainRegressor">TrainRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.html">TrainRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/TrainRegressor.scala">TrainRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automl" class="table-of-contents__link toc-highlight">AutoML</a><ul><li><a href="#findbestmodel" class="table-of-contents__link toc-highlight">FindBestModel</a></li><li><a href="#tunehyperparameters" class="table-of-contents__link toc-highlight">TuneHyperparameters</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#cleanmissingdata" class="table-of-contents__link toc-highlight">CleanMissingData</a></li><li><a href="#countselector" class="table-of-contents__link toc-highlight">CountSelector</a></li><li><a href="#featurize-1" class="table-of-contents__link toc-highlight">Featurize</a></li><li><a href="#valueindexer" class="table-of-contents__link toc-highlight">ValueIndexer</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#textfeaturizer" class="table-of-contents__link toc-highlight">TextFeaturizer</a></li></ul></li><li><a href="#isolation-forest" class="table-of-contents__link toc-highlight">Isolation Forest</a><ul><li><a href="#isolationforest" class="table-of-contents__link toc-highlight">IsolationForest</a></li></ul></li><li><a href="#nn" class="table-of-contents__link toc-highlight">NN</a><ul><li><a href="#conditionalknn" class="table-of-contents__link toc-highlight">ConditionalKNN</a></li><li><a href="#knn" class="table-of-contents__link toc-highlight">KNN</a></li></ul></li><li><a href="#recommendation" class="table-of-contents__link toc-highlight">Recommendation</a><ul><li><a href="#recommendationindexer-rankingevaluator-rankingadapter-and-rankingtrainvalidationsplit" class="table-of-contents__link toc-highlight">RecommendationIndexer, RankingEvaluator, RankingAdapter and RankingTrainValidationSplit</a></li><li><a href="#sar" class="table-of-contents__link toc-highlight">SAR</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#classbalancer" class="table-of-contents__link toc-highlight">ClassBalancer</a></li><li><a href="#multicolumnadapter" class="table-of-contents__link toc-highlight">MultiColumnAdapter</a></li><li><a href="#timer" class="table-of-contents__link toc-highlight">Timer</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#trainclassifier" class="table-of-contents__link toc-highlight">TrainClassifier</a></li><li><a href="#trainregressor" class="table-of-contents__link toc-highlight">TrainRegressor</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/estimators/estimators_lightgbm/index.html b/docs/next/Quick Examples/estimators/estimators_lightgbm/index.html
index 7a84dec2f2..9f102a9b2a 100644
--- a/docs/next/Quick Examples/estimators/estimators_lightgbm/index.html	
+++ b/docs/next/Quick Examples/estimators/estimators_lightgbm/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_lightgbm/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_lightgbm/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_lightgbm/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_lightgbm/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_lightgbm/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/estimators/estimators_lightgbm/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_lightgbm/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>LightGBM</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmClassifier = (new LightGBMClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;rawPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setObjective(&quot;binary&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLeafPredictionCol(&quot;leafPrediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesShapCol(&quot;featuresShap&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMClassifier">LightGBMClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.html">LightGBMClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMClassifier.scala">LightGBMClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmranker">LightGBMRanker<a href="#lightgbmranker" class="hash-link" aria-label="Direct link to LightGBMRanker" title="Direct link to LightGBMRanker">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRanker </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRanker</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;query&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRepartitionByGroupingColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRanker = (new LightGBMRanker()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setGroupCol(&quot;query&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRepartitionByGroupingColumn(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRanker">LightGBMRanker</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.html">LightGBMRanker</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRanker.scala">LightGBMRanker</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmregressor">LightGBMRegressor<a href="#lightgbmregressor" class="hash-link" aria-label="Direct link to LightGBMRegressor" title="Direct link to LightGBMRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lgbmRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.lightgbm._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lgbmRegressor = (new LightGBMRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;labels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDefaultListenPort(12402)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumLeaves(5)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumIterations(10))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lightgbm.html#module-synapse.ml.lightgbm.LightGBMRegressor">LightGBMRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.html">LightGBMRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/lightgbm/src/main/scala/com/microsoft/azure/synapse/ml/lightgbm/LightGBMRegressor.scala">LightGBMRegressor</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#lightgbmranker" class="table-of-contents__link toc-highlight">LightGBMRanker</a></li><li><a href="#lightgbmregressor" class="table-of-contents__link toc-highlight">LightGBMRegressor</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/estimators/estimators_vw/index.html b/docs/next/Quick Examples/estimators/estimators_vw/index.html
index b4850c47c2..8593a864b8 100644
--- a/docs/next/Quick Examples/estimators/estimators_vw/index.html	
+++ b/docs/next/Quick Examples/estimators/estimators_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/estimators/estimators_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/estimators/estimators_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/estimators/estimators_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/estimators/estimators_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/estimators/estimators_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/estimators/estimators_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/estimators/estimators_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitregressor">VowpalWabbitRegressor<a href="#vowpalwabbitregressor" class="hash-link" aria-label="Direct link to VowpalWabbitRegressor" title="Direct link to VowpalWabbitRegressor">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Y1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vwRegressor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumPasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;Y1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPredictionCol(&quot;pred&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vwRegressor = (new VowpalWabbitRegressor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setNumPasses(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--holdout_off --loss_function quantile -q :: -l 0.1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitRegressor">VowpalWabbitRegressor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.html">VowpalWabbitRegressor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitRegressor.scala">VowpalWabbitRegressor</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitcontextualbandit">VowpalWabbitContextualBandit<a href="#vowpalwabbitcontextualbandit" class="hash-link" aria-label="Direct link to VowpalWabbitContextualBandit" title="Direct link to VowpalWabbitContextualBandit">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitContextualBandit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPassThroughArgs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cost&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProbabilityCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setChosenActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;chosen_action&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSharedCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shared_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action_features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUseBarrierExecutionMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cb = (new VowpalWabbitContextualBandit()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPassThroughArgs(&quot;--cb_explore_adf --epsilon 0.2 --quiet&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLabelCol(&quot;cost&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProbabilityCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setChosenActionCol(&quot;chosen_action&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSharedCol(&quot;shared_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFeaturesCol(&quot;action_features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setUseBarrierExecutionMode(false))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitContextualBandit">VowpalWabbitContextualBandit</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.html">VowpalWabbitContextualBandit</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitContextualBandit.scala">VowpalWabbitContextualBandit</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vowpalwabbitregressor" class="table-of-contents__link toc-highlight">VowpalWabbitRegressor</a></li><li><a href="#vowpalwabbitcontextualbandit" class="table-of-contents__link toc-highlight">VowpalWabbitContextualBandit</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/transformers/transformers_cognitive/index.html b/docs/next/Quick Examples/transformers/transformers_cognitive/index.html
index a85851cdb5..1ccc4698dd 100644
--- a/docs/next/Quick Examples/transformers/transformers_cognitive/index.html	
+++ b/docs/next/Quick Examples/transformers/transformers_cognitive/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_cognitive/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_cognitive/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_cognitive/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_cognitive/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_cognitive/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/transformers/transformers_cognitive/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_cognitive/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="text-analytics">Text Analytics<a href="#text-analytics" class="hash-link" aria-label="Direct link to Text Analytics" title="Direct link to Text Analytics">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="entitydetector">EntityDetector<a href="#entitydetector" class="hash-link" aria-label="Direct link to EntityDetector" title="Direct link to EntityDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Microsoft released Windows 10&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">EntityDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.EntityDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;Microsoft released Windows 10&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;In 1975, Bill Gates III and Paul Allen founded the company.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val entity = (new EntityDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">entity.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.EntityDetector">EntityDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/EntityDetector.html">EntityDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">EntityDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="keyphraseextractor">KeyPhraseExtractor<a href="#keyphraseextractor" class="hash-link" aria-label="Direct link to KeyPhraseExtractor" title="Direct link to KeyPhraseExtractor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello world. This is some input text that I love.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">KeyPhraseExtractor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lang&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.KeyPhraseExtractor</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val keyPhrase = (new KeyPhraseExtractor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">keyPhrase.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.KeyPhraseExtractor">KeyPhraseExtractor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/KeyPhraseExtractor.html">KeyPhraseExtractor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">KeyPhraseExtractor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="languagedetector">LanguageDetector<a href="#languagedetector" class="hash-link" aria-label="Direct link to LanguageDetector" title="Direct link to LanguageDetector">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello World&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bonjour tout le monde&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;你好&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;:) :( :D&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LanguageDetector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.LanguageDetector</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Hello World&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;Bonjour tout le monde&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;:) :( :D&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val language = (new LanguageDetector()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">language.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.LanguageDetector">LanguageDetector</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/LanguageDetector.html">LanguageDetector</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">LanguageDetector</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ner">NER<a href="#ner" class="hash-link" aria-label="Direct link to NER" title="Direct link to NER">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I had a wonderful trip to Seattle last week.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I visited Space Needle 2 times.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">NER</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;replies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.NER</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;I had a wonderful trip to Seattle last week.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;, &quot;I visited Space Needle 2 times.&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ner = (new NER()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ner.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.NER">NER</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/NER.html">NER</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">NER</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pii">PII<a href="#pii" class="hash-link" aria-label="Direct link to PII" title="Direct link to PII">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;My SSN is 859-98-0987&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Is 998.214.865-68 your Brazilian CPF number?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PII</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.PII</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1&quot;, &quot;en&quot;, &quot;My SSN is 859-98-0987&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;2&quot;, &quot;en&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;Your ABA number - 111000025 - is the first 9 digits in the lower left hand corner of your personal check.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;3&quot;, &quot;en&quot;, &quot;Is 998.214.865-68 your Brazilian CPF number?&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;id&quot;, &quot;language&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pii = (new PII()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;response&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pii.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.PII">PII</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/PII.html">PII</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">PII</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textsentiment">TextSentiment<a href="#textsentiment" class="hash-link" aria-label="Direct link to TextSentiment" title="Direct link to TextSentiment">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">textKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.text.TextSentiment</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, &quot;Hello world. This is some input text that I love.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;fr&quot;, &quot;Bonjour tout le monde&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;es&quot;, &quot;La carretera estaba atascada. Había mucho tráfico el día de ayer.&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, &quot;ich bin ein berliner&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (null, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;en&quot;, null)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;lang&quot;, &quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sentiment = (new TextSentiment()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(textKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;lang&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelVersion(&quot;latest&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setShowStats(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;replies&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sentiment.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.TextSentiment">TextSentiment</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TextSentiment.html">TextSentiment</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextAnalytics.scala">TextSentiment</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="translator">Translator<a href="#translator" class="hash-link" aria-label="Direct link to Translator" title="Direct link to Translator">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="translate">Translate<a href="#translate" class="hash-link" aria-label="Direct link to Translate" title="Direct link to Translate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Bye&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Translate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;zh-Hans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;, &quot;Bye&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translate = (new Translate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(Seq(&quot;zh-Hans&quot;, &quot;fr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(translate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, flatten(col(&quot;translation.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translation&quot;, col(&quot;translation.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;translation&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Translate">Translate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Translate.html">Translate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Translate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="transliterate">Transliterate<a href="#transliterate" class="hash-link" aria-label="Direct link to Transliterate" title="Direct link to Transliterate">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;こんにちは&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;さようなら&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transliterate </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Transliterate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ja&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Jpan&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToScript</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Latn&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;script&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;こんにちは&quot;, &quot;さようなら&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transliterate = (new Transliterate()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;ja&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromScript(&quot;Jpan&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToScript(&quot;Latn&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(transliterate</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;text&quot;, col(&quot;result.text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;script&quot;, col(&quot;result.script&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;text&quot;, &quot;script&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Transliterate">Transliterate</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Transliterate.html">Transliterate</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Transliterate</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detect">Detect<a href="#detect" class="hash-link" aria-label="Direct link to Detect" title="Direct link to Detect">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detect </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Detect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.Detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detect = (new Detect()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(detect</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;language&quot;, col(&quot;result.language&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;language&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.Detect">Detect</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/Detect.html">Detect</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">Detect</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="breaksentence">BreakSentence<a href="#breaksentence" class="hash-link" aria-label="Direct link to BreakSentence" title="Direct link to BreakSentence">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain">  spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hello, what is your name?&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">breakSentence </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BreakSentence</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentLen&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.BreakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;Hello, what is your name?&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val breakSentence = (new BreakSentence()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(breakSentence</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;sentLen&quot;, flatten(col(&quot;result.sentLen&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;sentLen&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BreakSentence">BreakSentence</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BreakSentence.html">BreakSentence</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">BreakSentence</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionarylookup">DictionaryLookup<a href="#dictionarylookup" class="hash-link" aria-label="Direct link to DictionaryLookup" title="Direct link to DictionaryLookup">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryLookup </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryLookup</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.translations&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translations.normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;normalizedTarget&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(&quot;fly&quot;)).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryLookup = (new DictionaryLookup()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryLookup</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;translations&quot;, flatten(col(&quot;result.translations&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;normalizedTarget&quot;, col(&quot;translations.normalizedTarget&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;normalizedTarget&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryLookup">DictionaryLookup</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryLookup.html">DictionaryLookup</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryLookup</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dictionaryexamples">DictionaryExamples<a href="#dictionaryexamples" class="hash-link" aria-label="Direct link to DictionaryExamples" title="Direct link to DictionaryExamples">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;volar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dictionaryExamples </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DictionaryExamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFromLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setToLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;es&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextAndTranslationCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;textAndTranslation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;examples&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.{DictionaryExamples, TextAndTranslation}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, flatten}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(List(TextAndTranslation(&quot;fly&quot;, &quot;volar&quot;))).toDF(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dictionaryExamples = (new DictionaryExamples()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFromLanguage(&quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setToLanguage(&quot;es&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTextAndTranslationCol(&quot;textAndTranslation&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">(dictionaryExamples</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;examples&quot;, flatten(col(&quot;result.examples&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(&quot;examples&quot;)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DictionaryExamples">DictionaryExamples</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DictionaryExamples.html">DictionaryExamples</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/TextTranslator.scala">DictionaryExamples</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="documenttranslator">DocumentTranslator<a href="#documenttranslator" class="hash-link" aria-label="Direct link to DocumentTranslator" title="Direct link to DocumentTranslator">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translator-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">translatorName </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;TRANSLATOR_NAME&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-translator&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">documentTranslator </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DocumentTranslator</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">translatorName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSourceUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sourceUrl&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;targets&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;translationStatus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.translate.DocumentTranslator</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorKey = sys.env.getOrElse(&quot;TRANSLATOR_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val translatorName = sys.env.getOrElse(&quot;TRANSLATOR_NAME&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val documentTranslator = (new DocumentTranslator()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(translatorKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(translatorName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSourceUrlCol(&quot;sourceUrl&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setTargetsCol(&quot;targets&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;translationStatus&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DocumentTranslator">DocumentTranslator</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.html">DocumentTranslator</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/DocumentTranslator.scala">DocumentTranslator</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="computer-vision">Computer Vision<a href="#computer-vision" class="hash-link" aria-label="Direct link to Computer Vision" title="Direct link to Computer Vision">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ocr">OCR<a href="#ocr" class="hash-link" aria-label="Direct link to OCR" title="Direct link to OCR">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">OCR</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetectOrientation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.OCR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ocr = (new OCR()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetectOrientation(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ocr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.OCR">OCR</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/OCR.html">OCR</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">OCR</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeimage">AnalyzeImage<a href="#analyzeimage" class="hash-link" aria-label="Direct link to AnalyzeImage" title="Direct link to AnalyzeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setVisualFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Categories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Description&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ImageType&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Color&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Adult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Objects&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Brands&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDetails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Landmarks&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.AnalyzeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;, &quot;en&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;, null),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;, &quot;en&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;, &quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ai = (new AnalyzeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguageCol(&quot;language&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setVisualFeatures(Seq(&quot;Categories&quot;, &quot;Tags&quot;, &quot;Description&quot;, &quot;Faces&quot;, &quot;ImageType&quot;, &quot;Color&quot;, &quot;Adult&quot;, &quot;Objects&quot;, &quot;Brands&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setDetails(Seq(&quot;Celebrities&quot;, &quot;Landmarks&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ai.transform(df).select(&quot;url&quot;, &quot;features&quot;).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeImage">AnalyzeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeImage.html">AnalyzeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">AnalyzeImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizetext">RecognizeText<a href="#recognizetext" class="hash-link" aria-label="Direct link to RecognizeText" title="Direct link to RecognizeText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Printed&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rt = (new RecognizeText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMode(&quot;Printed&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeText">RecognizeText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeText.html">RecognizeText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="readimage">ReadImage<a href="#readimage" class="hash-link" aria-label="Direct link to ReadImage" title="Direct link to ReadImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ReadImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ocr&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.ReadImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test2.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/OCR/test3.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ri = (new ReadImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ocr&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ri.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.ReadImage">ReadImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ReadImage.html">ReadImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">ReadImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="recognizedomainspecificcontent">RecognizeDomainSpecificContent<a href="#recognizedomainspecificcontent" class="hash-link" aria-label="Direct link to RecognizeDomainSpecificContent" title="Direct link to RecognizeDomainSpecificContent">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">RecognizeDomainSpecificContent</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebrities&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;celebs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.RecognizeDomainSpecificContent</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val celeb = (new RecognizeDomainSpecificContent()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModel(&quot;celebrities&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;celebs&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">celeb.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.RecognizeDomainSpecificContent">RecognizeDomainSpecificContent</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/RecognizeDomainSpecificContent.html">RecognizeDomainSpecificContent</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">RecognizeDomainSpecificContent</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatethumbnails">GenerateThumbnails<a href="#generatethumbnails" class="hash-link" aria-label="Direct link to GenerateThumbnails" title="Direct link to GenerateThumbnails">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GenerateThumbnails</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setHeight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSmartCropping</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;thumbnails&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.GenerateThumbnails</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val gt = (new GenerateThumbnails()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setHeight(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setWidth(50)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSmartCropping(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;thumbnails&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">gt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GenerateThumbnails">GenerateThumbnails</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GenerateThumbnails.html">GenerateThumbnails</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">GenerateThumbnails</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tagimage">TagImage<a href="#tagimage" class="hash-link" aria-label="Direct link to TagImage" title="Direct link to TagImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TagImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tags&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.TagImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ti = (new TagImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;tags&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ti.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.TagImage">TagImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/TagImage.html">TagImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">TagImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="describeimage">DescribeImage<a href="#describeimage" class="hash-link" aria-label="Direct link to DescribeImage" title="Direct link to DescribeImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DescribeImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxCandidates</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;descriptions&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.vision.DescribeImage</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val di = (new DescribeImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setMaxCandidates(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;descriptions&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">di.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.DescribeImage">DescribeImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DescribeImage.html">DescribeImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/ComputerVision.scala">DescribeImage</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer">Form Recognizer<a href="#form-recognizer" class="hash-link" aria-label="Direct link to Form Recognizer" title="Direct link to Form Recognizer">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzelayout">AnalyzeLayout<a href="#analyzelayout" class="hash-link" aria-label="Direct link to AnalyzeLayout" title="Direct link to AnalyzeLayout">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeLayout</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.readResults.lines&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lines.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;layout.analyzeResult.pageResults.tables&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;readLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pageLayout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeLayout</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeLayout = (new AnalyzeLayout()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeLayout.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeLayout">AnalyzeLayout</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeLayout.html">AnalyzeLayout</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeLayout</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzereceipts">AnalyzeReceipts<a href="#analyzereceipts" class="hash-link" aria-label="Direct link to AnalyzeReceipts" title="Direct link to AnalyzeReceipts">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;receipts&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeReceipts</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/receipt1.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeReceipts = (new AnalyzeReceipts()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;receipts&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeReceipts.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeReceipts">AnalyzeReceipts</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeReceipts.html">AnalyzeReceipts</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeReceipts</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzebusinesscards">AnalyzeBusinessCards<a href="#analyzebusinesscards" class="hash-link" aria-label="Direct link to AnalyzeBusinessCards" title="Direct link to AnalyzeBusinessCards">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;businessCards&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeBusinessCards</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/business_card.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeBusinessCards = (new AnalyzeBusinessCards()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;businessCards&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeBusinessCards.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeBusinessCards">AnalyzeBusinessCards</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeBusinessCards.html">AnalyzeBusinessCards</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeBusinessCards</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeinvoices">AnalyzeInvoices<a href="#analyzeinvoices" class="hash-link" aria-label="Direct link to AnalyzeInvoices" title="Direct link to AnalyzeInvoices">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeInvoices</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;invoices.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeInvoices</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeInvoices = (new AnalyzeInvoices()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;invoices&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeInvoices.transform(imageD4).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeInvoices">AnalyzeInvoices</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeInvoices.html">AnalyzeInvoices</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeInvoices</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzeiddocuments">AnalyzeIDDocuments<a href="#analyzeiddocuments" class="hash-link" aria-label="Direct link to AnalyzeIDDocuments" title="Direct link to AnalyzeIDDocuments">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeIDDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ids.analyzeResult.documentResults.fields&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;documents&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeIDDocuments</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/id1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeIDDocuments = (new AnalyzeIDDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;ids&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeIDDocuments.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeIDDocuments">AnalyzeIDDocuments</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeIDDocuments.html">AnalyzeIDDocuments</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeIDDocuments</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzecustommodel">AnalyzeCustomModel<a href="#analyzecustommodel" class="hash-link" aria-label="Direct link to AnalyzeCustomModel" title="Direct link to AnalyzeCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                 </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output.analyzeResult.pageResults.keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.key.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs.value.text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> create_map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keys&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;keyValuePairs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/invoice2.png&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeCustomModel = (new AnalyzeCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;output&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeCustomModel.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeCustomModel">AnalyzeCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeCustomModel.html">AnalyzeCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">AnalyzeCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getcustommodel">GetCustomModel<a href="#getcustommodel" class="hash-link" aria-label="Direct link to GetCustomModel" title="Direct link to GetCustomModel">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">modelId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># put your own modelId here</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GetCustomModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">modelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIncludeKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">getCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.ModelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;model.TrainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelInfo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trainResult&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.GetCustomModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val modelId = &quot;02bc2f58-2beb-4ae3-84fb-08f011b2f7b8&quot; // put your own modelId here</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getCustomModel = (new GetCustomModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setModelId(modelId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIncludeKeys(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;model&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getCustomModel.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.GetCustomModel">GetCustomModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GetCustomModel.html">GetCustomModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">GetCustomModel</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="listcustommodels">ListCustomModels<a href="#listcustommodels" class="hash-link" aria-label="Direct link to ListCustomModels" title="Direct link to ListCustomModels">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">emptyDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ListCustomModels</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;full&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">listCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">emptyDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;models.modelList.modelId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">       </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;modelIds&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.ListCustomModels</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val emptyDf = Seq(&quot;&quot;).toDF()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val listCustomModels = (new ListCustomModels()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOp(&quot;full&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;models&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">listCustomModels.transform(emptyDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-mmlspark.cognitive.ListCustomModels">ListCustomModels</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/ListCustomModels.html">ListCustomModels</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizer.scala">ListCustomModels</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="form-recognizer-v3">Form Recognizer V3<a href="#form-recognizer-v3" class="hash-link" aria-label="Direct link to Form Recognizer V3" title="Direct link to Form Recognizer V3">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="analyzedocument">AnalyzeDocument<a href="#analyzedocument" class="hash-link" aria-label="Direct link to AnalyzeDocument" title="Direct link to AnalyzeDocument">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imageDf </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AnalyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)"># For supported prebuilt models, please go to documentation page for details</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrebuiltModelId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prebuilt-layout&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">analyzeDocument</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imageDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> flatten</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result.analyzeResult.tables.cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells.content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;source&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;result&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;content&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cells&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.form.AnalyzeDocument</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imageDf = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/FormRecognizer/layout1.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val analyzeDocument = (new AnalyzeDocument()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPrebuiltModelId(&quot;prebuilt-layout&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;source&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;result&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setConcurrency(5))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">analyzeDocument.transform(imageDf).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AnalyzeDocument">AnalyzeDocument</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AnalyzeDocument.html">AnalyzeDocument</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/FormRecognizerV3.scala">AnalyzeDocument</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="anomaly-detection">Anomaly Detection<a href="#anomaly-detection" class="hash-link" aria-label="Direct link to Anomaly Detection" title="Direct link to Anomaly Detection">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectlastanomaly">DetectLastAnomaly<a href="#detectlastanomaly" class="hash-link" aria-label="Direct link to DetectLastAnomaly" title="Direct link to DetectLastAnomaly">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> lit</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectLastAnomaly</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;errors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectLastAnomaly</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.{col, collect_list, lit, sort_array, struct}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dla = (new DetectLastAnomaly()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;errors&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dla.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectLastAnomaly">DetectLastAnomaly</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectLastAnomaly.html">DetectLastAnomaly</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectLastAnomaly</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectanomalies">DetectAnomalies<a href="#detectanomalies" class="hash-link" aria-label="Direct link to DetectAnomalies" title="Direct link to DetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> struct</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">groupBy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">agg</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sort_array</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">collect_list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSeriesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inputs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.DetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 90000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;timestamp&quot;, &quot;value&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;group&quot;, lit(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .withColumn(&quot;inputs&quot;, struct(col(&quot;timestamp&quot;), col(&quot;value&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .groupBy(col(&quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .agg(sort_array(collect_list(col(&quot;inputs&quot;))).alias(&quot;inputs&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val da = (new DetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSeriesCol(&quot;inputs&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">da.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectAnomalies">DetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectAnomalies.html">DetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">DetectAnomalies</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simpledetectanomalies">SimpleDetectAnomalies<a href="#simpledetectanomalies" class="hash-link" aria-label="Direct link to SimpleDetectAnomalies" title="Direct link to SimpleDetectAnomalies">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">anomalyKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ANOMALY_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomaly-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">826.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">799.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">890.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-04-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">900.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-05-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">766.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-06-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">805.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-07-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">821.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-08-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-09-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">883.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-10-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">898.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-11-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">957.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1972-12-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">924.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-01-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">881.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-02-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">837.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1973-03-01T00:00:00Z&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">90000.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;timestamp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;value&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleDetectAnomalies</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">anomalyKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;westus2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;anomalies&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGroupbyCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;group&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setGranularity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;monthly&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.anomaly.SimpleDetectAnomalies</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val anomalyKey = sys.env.getOrElse(&quot;ANOMALY_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val baseSeq = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-01-01T00:00:00Z&quot;, 826.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-02-01T00:00:00Z&quot;, 799.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-03-01T00:00:00Z&quot;, 890.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-04-01T00:00:00Z&quot;, 900.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-05-01T00:00:00Z&quot;, 766.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-06-01T00:00:00Z&quot;, 805.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-07-01T00:00:00Z&quot;, 821.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-08-01T00:00:00Z&quot;, 20000.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-09-01T00:00:00Z&quot;, 883.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-10-01T00:00:00Z&quot;, 898.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-11-01T00:00:00Z&quot;, 957.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1972-12-01T00:00:00Z&quot;, 924.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-01-01T00:00:00Z&quot;, 881.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-02-01T00:00:00Z&quot;, 837.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (&quot;1973-03-01T00:00:00Z&quot;, 9000.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (baseSeq.map(p =&gt; (p._1, p._2, 1.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .++(baseSeq.map(p =&gt; (p._1, p._2, 2.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;timestamp&quot;, &quot;value&quot;, &quot;group&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sda = (new SimpleDetectAnomalies()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(anomalyKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;westus2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;anomalies&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGroupbyCol(&quot;group&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setGranularity(&quot;monthly&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SimpleDetectAnomalies">SimpleDetectAnomalies</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SimpleDetectAnomalies.html">SimpleDetectAnomalies</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AnomalyDetection.scala">SimpleDetectAnomalies</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="face">Face<a href="#face" class="hash-link" aria-label="Direct link to Face" title="Direct link to Face">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="detectface">DetectFace<a href="#detectface" class="hash-link" aria-label="Direct link to DetectFace" title="Direct link to DetectFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;gender&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;headPose&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;smile&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;facialHair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;glasses&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;emotion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hair&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;makeup&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occlusion&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accessories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blur&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exposure&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;noise&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.DetectFace</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val face = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;face&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;age&quot;, &quot;gender&quot;, &quot;headPose&quot;, &quot;smile&quot;, &quot;facialHair&quot;, &quot;glasses&quot;, &quot;emotion&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hair&quot;, &quot;makeup&quot;, &quot;occlusion&quot;, &quot;accessories&quot;, &quot;blur&quot;, &quot;exposure&quot;, &quot;noise&quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">face.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.DetectFace">DetectFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/DetectFace.html">DetectFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">DetectFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="findsimilarface">FindSimilarFace<a href="#findsimilarface" class="hash-link" aria-label="Direct link to FindSimilarFace" title="Direct link to FindSimilarFace">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FindSimilarFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;similar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, FindSimilarFace}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val findSimilar = (new FindSimilarFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;similar&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdCol(&quot;id&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">findSimilar.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.FindSimilarFace">FindSimilarFace</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/FindSimilarFace.html">FindSimilarFace</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">FindSimilarFace</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="groupfaces">GroupFaces<a href="#groupfaces" class="hash-link" aria-label="Direct link to GroupFaces" title="Direct link to GroupFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIds </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">row</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;id&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> row </span><span class="token keyword">in</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">GroupFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;grouping&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIds</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, GroupFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIds = faceIdDF.collect().map(row =&gt; row.getAs[String](&quot;id&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val group = (new GroupFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;grouping&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIds(faceIds))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">group.transform(faceIdDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.GroupFaces">GroupFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/GroupFaces.html">GroupFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">GroupFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="identifyfaces">IdentifyFaces<a href="#identifyfaces" class="hash-link" aria-label="Direct link to IdentifyFaces" title="Direct link to IdentifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pgId </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">identifyFaces </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IdentifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceIdsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPersonGroupId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pgId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;identified_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.IdentifyFaces</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val pgId = &quot;PUT_YOUR_PERSON_GROUP_ID&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val identifyFaces = (new IdentifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceIdsCol(&quot;faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setPersonGroupId(pgId)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;identified_faces&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.IdentifyFaces">IdentifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/IdentifyFaces.html">IdentifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">IdentifyFaces</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="verifyfaces">VerifyFaces<a href="#verifyfaces" class="hash-link" aria-label="Direct link to VerifyFaces" title="Direct link to VerifyFaces">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">detector </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DetectFace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setImageUrlCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceId</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceLandmarks</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setReturnFaceAttributes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF </span><span class="token operator">=</span><span class="token plain"> detector</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;detected_faces&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getItem</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">alias</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">faceIdDF2 </span><span class="token operator">=</span><span class="token plain"> faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> lit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">take</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">asDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;faceId1&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VerifyFaces</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;same&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId1Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFaceId2Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;faceId2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">faceIdDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.face.{DetectFace, VerifyFaces}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test1.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test2.jpg&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/DSIR/test3.jpg&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val detector = (new DetectFace()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setImageUrlCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;detected_faces&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceId(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceLandmarks(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setReturnFaceAttributes(Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF = (detector.transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .select(col(&quot;detected_faces&quot;).getItem(0).getItem(&quot;faceId&quot;).alias(&quot;faceId1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .cache())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val faceIdDF2 = faceIdDF.withColumn(&quot;faceId2&quot;, lit(faceIdDF.take(1).head.getString(0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val verify = (new VerifyFaces()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;same&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId1Col(&quot;faceId1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFaceId2Col(&quot;faceId2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">verify.transform(faceIdDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.VerifyFaces">VerifyFaces</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/VerifyFaces.html">VerifyFaces</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/Face.scala">VerifyFaces</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="speech-to-text">Speech To Text<a href="#speech-to-text" class="hash-link" aria-label="Direct link to Speech To Text" title="Direct link to Speech To Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotext">SpeechToText<a href="#speechtotext" class="hash-link" aria-label="Direct link to SpeechToText" title="Direct link to SpeechToText">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">link </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">audioBytes </span><span class="token operator">=</span><span class="token plain"> requests</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">link</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">content</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">audioBytes</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToText</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;audio&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFormat</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;simple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToText</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val audioBytes = IOUtils.toByteArray(new URL(&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/test1.wav&quot;).openStream())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Tuple1(audioBytes)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stt = (new SpeechToText()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;audio&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setFormat(&quot;simple&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToText">SpeechToText</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.html">SpeechToText</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToText.scala">SpeechToText</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="speechtotextsdk">SpeechToTextSDK<a href="#speechtotextsdk" class="hash-link" aria-label="Direct link to SpeechToTextSDK" title="Direct link to SpeechToTextSDK">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> requests</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cognitiveKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;COGNITIVE_API_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cognitive-api-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                           </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SpeechToTextSDK</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cognitiveKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAudioDataCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setProfanity</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Masked&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.speech.SpeechToTextSDK</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.commons.compress.utils.IOUtils</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import java.net.URL</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cognitiveKey = sys.env.getOrElse(&quot;COGNITIVE_API_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df: DataFrame = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  &quot;https://mmlspark.blob.core.windows.net/datasets/Speech/audio2.wav&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val speech_to_text = (new SpeechToTextSDK()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(cognitiveKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLocation(&quot;eastus&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setAudioDataCol(&quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setLanguage(&quot;en-US&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setProfanity(&quot;Masked&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">speech_to_text.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.SpeechToTextSDK">SpeechToTextSDK</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.html">SpeechToTextSDK</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/SpeechToTextSDK.scala">SpeechToTextSDK</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="azure-search">Azure Search<a href="#azure-search" class="hash-link" aria-label="Direct link to Azure Search" title="Direct link to Azure Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azuresearch">AzureSearch<a href="#azuresearch" class="hash-link" aria-label="Direct link to AzureSearch" title="Direct link to AzureSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">azureSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;AZURE_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;azure-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testServiceName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;mmlspark-azure-search&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">indexName </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;test-website&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    json_str </span><span class="token operator">=</span><span class="token plain"> </span><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">       {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;name&quot;: &quot;%s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">           &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">               {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                   &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                    &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">                }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">            ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">        }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token triple-quoted-string string" style="color:rgb(230, 219, 116)">    &quot;&quot;&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> json_str </span><span class="token operator">%</span><span class="token plain"> indexName</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;upload&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;file3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;id&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fileName&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">AddDocuments</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;err&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setIndexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setActionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">writeToAzureSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    subscriptionKey</span><span class="token operator">=</span><span class="token plain">azureSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    actionCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;searchAction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    serviceName</span><span class="token operator">=</span><span class="token plain">testServiceName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    indexJson</span><span class="token operator">=</span><span class="token plain">createSimpleIndexJson</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">indexName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.search.{AddDocuments, AzureSearchWriter}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val azureSearchKey = sys.env.getOrElse(&quot;AZURE_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testServiceName = &quot;mmlspark-azure-search&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val indexName = &quot;test-website&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">def createSimpleIndexJson(indexName: String) = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  s&quot;&quot;&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |{</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;name&quot;: &quot;$indexName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    &quot;fields&quot;: [</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;name&quot;: &quot;id&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;key&quot;: true,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |        &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;fileName&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;searchable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    },</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;name&quot;: &quot;text&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;type&quot;: &quot;Edm.String&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;filterable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;sortable&quot;: false,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |      &quot;facetable&quot;: false</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |    ]</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">     |  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;&quot;.stripMargin</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = ((0 until 4)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .map(i =&gt; (&quot;upload&quot;, s&quot;$i&quot;, s&quot;file$i&quot;, s&quot;text$i&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .toDF(&quot;searchAction&quot;, &quot;id&quot;, &quot;fileName&quot;, &quot;text&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ad = (new AddDocuments()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(azureSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setServiceName(testServiceName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setErrorCol(&quot;err&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setIndexName(indexName)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setActionCol(&quot;searchAction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ad.transform(df).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">AzureSearchWriter.write(df,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Map(&quot;subscriptionKey&quot; -&gt; azureSearchKey,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;actionCol&quot; -&gt; &quot;searchAction&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;serviceName&quot; -&gt; testServiceName,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;indexJson&quot; -&gt; createSimpleIndexJson(indexName)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.AzureSearch">AzureSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.html">AzureSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/AzureSearch.scala">AzureSearch</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="bing-image-search">Bing Image Search<a href="#bing-image-search" class="hash-link" aria-label="Direct link to Bing Image Search" title="Direct link to Bing Image Search">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="bingimagesearch">BingImageSearch<a href="#bingimagesearch" class="hash-link" aria-label="Direct link to BingImageSearch" title="Direct link to BingImageSearch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearchKey </span><span class="token operator">=</span><span class="token plain"> os</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">environ</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;BING_SEARCH_KEY&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getSecret</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bing-search-key&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Number of images Bing will return per query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">imgsPerBatch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># A list of offsets, used to page into the search results</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">offsets </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">i</span><span class="token operator">*</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> i </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since web content is our data, we create a dataframe with options on that data: offsets</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingParameters </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">offsets</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Run the Bing Image Search service with our text query</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingSearchKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOffsetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;offset&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setQuery</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Martin Luther King Jr. quotes&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCount</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">imgsPerBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">              </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls </span><span class="token operator">=</span><span class="token plain"> BingImageSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getUrlTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;url&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># This displays the full results returned</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Since we have two services, they are put into a pipeline</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline </span><span class="token operator">=</span><span class="token plain"> PipelineModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stages</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">bingSearch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> getUrls</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># Show the results of your search: image URLs</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">pipeline</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">bingParameters</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.services.bing.BingImageSearch</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearchKey = sys.env.getOrElse(&quot;BING_SEARCH_KEY&quot;, None)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Number of images Bing will return per query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val imgsPerBatch = 10</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// A list of offsets, used to page into the search results</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val offsets = (0 until 100).map(i =&gt; i * imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Since web content is our data, we create a dataframe with options on that data: offsets</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingParameters = Seq(offsets).toDF(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Run the Bing Image Search service with our text query</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val bingSearch = (new BingImageSearch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setSubscriptionKey(bingSearchKey)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOffsetCol(&quot;offset&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setQuery(&quot;Martin Luther King Jr. quotes&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setCount(imgsPerBatch)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  .setOutputCol(&quot;images&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Transformer that extracts and flattens the richly structured output of Bing Image Search into a simple URL column</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val getUrls = BingImageSearch.getUrlTransformer(&quot;images&quot;, &quot;url&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// This displays the full results returned</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">bingSearch.transform(bingParameters).show()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">// Show the results of your search: image URLs</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">getUrls.transform(bingSearch.transform(bingParameters)).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.cognitive.html#module-synapse.ml.cognitive.BingImageSearch">BingImageSearch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.html">BingImageSearch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/cognitive/src/main/scala/com/microsoft/azure/synapse/ml/cognitive/BingImageSearch.scala">BingImageSearch</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#text-analytics" class="table-of-contents__link toc-highlight">Text Analytics</a><ul><li><a href="#entitydetector" class="table-of-contents__link toc-highlight">EntityDetector</a></li><li><a href="#keyphraseextractor" class="table-of-contents__link toc-highlight">KeyPhraseExtractor</a></li><li><a href="#languagedetector" class="table-of-contents__link toc-highlight">LanguageDetector</a></li><li><a href="#ner" class="table-of-contents__link toc-highlight">NER</a></li><li><a href="#pii" class="table-of-contents__link toc-highlight">PII</a></li><li><a href="#textsentiment" class="table-of-contents__link toc-highlight">TextSentiment</a></li></ul></li><li><a href="#translator" class="table-of-contents__link toc-highlight">Translator</a><ul><li><a href="#translate" class="table-of-contents__link toc-highlight">Translate</a></li><li><a href="#transliterate" class="table-of-contents__link toc-highlight">Transliterate</a></li><li><a href="#detect" class="table-of-contents__link toc-highlight">Detect</a></li><li><a href="#breaksentence" class="table-of-contents__link toc-highlight">BreakSentence</a></li><li><a href="#dictionarylookup" class="table-of-contents__link toc-highlight">DictionaryLookup</a></li><li><a href="#dictionaryexamples" class="table-of-contents__link toc-highlight">DictionaryExamples</a></li><li><a href="#documenttranslator" class="table-of-contents__link toc-highlight">DocumentTranslator</a></li></ul></li><li><a href="#computer-vision" class="table-of-contents__link toc-highlight">Computer Vision</a><ul><li><a href="#ocr" class="table-of-contents__link toc-highlight">OCR</a></li><li><a href="#analyzeimage" class="table-of-contents__link toc-highlight">AnalyzeImage</a></li><li><a href="#recognizetext" class="table-of-contents__link toc-highlight">RecognizeText</a></li><li><a href="#readimage" class="table-of-contents__link toc-highlight">ReadImage</a></li><li><a href="#recognizedomainspecificcontent" class="table-of-contents__link toc-highlight">RecognizeDomainSpecificContent</a></li><li><a href="#generatethumbnails" class="table-of-contents__link toc-highlight">GenerateThumbnails</a></li><li><a href="#tagimage" class="table-of-contents__link toc-highlight">TagImage</a></li><li><a href="#describeimage" class="table-of-contents__link toc-highlight">DescribeImage</a></li></ul></li><li><a href="#form-recognizer" class="table-of-contents__link toc-highlight">Form Recognizer</a><ul><li><a href="#analyzelayout" class="table-of-contents__link toc-highlight">AnalyzeLayout</a></li><li><a href="#analyzereceipts" class="table-of-contents__link toc-highlight">AnalyzeReceipts</a></li><li><a href="#analyzebusinesscards" class="table-of-contents__link toc-highlight">AnalyzeBusinessCards</a></li><li><a href="#analyzeinvoices" class="table-of-contents__link toc-highlight">AnalyzeInvoices</a></li><li><a href="#analyzeiddocuments" class="table-of-contents__link toc-highlight">AnalyzeIDDocuments</a></li><li><a href="#analyzecustommodel" class="table-of-contents__link toc-highlight">AnalyzeCustomModel</a></li><li><a href="#getcustommodel" class="table-of-contents__link toc-highlight">GetCustomModel</a></li><li><a href="#listcustommodels" class="table-of-contents__link toc-highlight">ListCustomModels</a></li></ul></li><li><a href="#form-recognizer-v3" class="table-of-contents__link toc-highlight">Form Recognizer V3</a><ul><li><a href="#analyzedocument" class="table-of-contents__link toc-highlight">AnalyzeDocument</a></li></ul></li><li><a href="#anomaly-detection" class="table-of-contents__link toc-highlight">Anomaly Detection</a><ul><li><a href="#detectlastanomaly" class="table-of-contents__link toc-highlight">DetectLastAnomaly</a></li><li><a href="#detectanomalies" class="table-of-contents__link toc-highlight">DetectAnomalies</a></li><li><a href="#simpledetectanomalies" class="table-of-contents__link toc-highlight">SimpleDetectAnomalies</a></li></ul></li><li><a href="#face" class="table-of-contents__link toc-highlight">Face</a><ul><li><a href="#detectface" class="table-of-contents__link toc-highlight">DetectFace</a></li><li><a href="#findsimilarface" class="table-of-contents__link toc-highlight">FindSimilarFace</a></li><li><a href="#groupfaces" class="table-of-contents__link toc-highlight">GroupFaces</a></li><li><a href="#identifyfaces" class="table-of-contents__link toc-highlight">IdentifyFaces</a></li><li><a href="#verifyfaces" class="table-of-contents__link toc-highlight">VerifyFaces</a></li></ul></li><li><a href="#speech-to-text" class="table-of-contents__link toc-highlight">Speech To Text</a><ul><li><a href="#speechtotext" class="table-of-contents__link toc-highlight">SpeechToText</a></li><li><a href="#speechtotextsdk" class="table-of-contents__link toc-highlight">SpeechToTextSDK</a></li></ul></li><li><a href="#azure-search" class="table-of-contents__link toc-highlight">Azure Search</a><ul><li><a href="#azuresearch" class="table-of-contents__link toc-highlight">AzureSearch</a></li></ul></li><li><a href="#bing-image-search" class="table-of-contents__link toc-highlight">Bing Image Search</a><ul><li><a href="#bingimagesearch" class="table-of-contents__link toc-highlight">BingImageSearch</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/transformers/transformers_core/index.html b/docs/next/Quick Examples/transformers/transformers_core/index.html
index e7a3372cd4..af281c78ea 100644
--- a/docs/next/Quick Examples/transformers/transformers_core/index.html	
+++ b/docs/next/Quick Examples/transformers/transformers_core/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_core/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_core/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_core/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_core/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_core/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/transformers/transformers_core/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_core/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h2 class="anchor anchorWithStickyNavbar_LWe7" id="explainers">Explainers<a href="#explainers" class="hash-link" aria-label="Direct link to Explainers" title="Direct link to Explainers">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imagelime">ImageLIME<a href="#imagelime" class="hash-link" aria-label="Direct link to ImageLIME" title="Direct link to ImageLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new ImageLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageLIME">ImageLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.html">ImageLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageLIME.scala">ImageLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="imageshap">ImageSHAP<a href="#imageshap" class="hash-link" aria-label="Direct link to ImageSHAP" title="Direct link to ImageSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ImageSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shaps&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSuperpixelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;superpixels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCellSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">150.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">500</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClassesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;top2pred&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new ImageSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shaps&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSuperpixelCol(&quot;superpixels&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setCellSize(150.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModifier(50.0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(500)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClassesCol(&quot;top2pred&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.ImageSHAP">ImageSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.html">ImageSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/ImageSHAP.scala">ImageSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularlime">TabularLIME<a href="#tabularlime" class="hash-link" aria-label="Direct link to TabularLIME" title="Direct link to TabularLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKernelWidth</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.001</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-6.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (-5.0, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (5.0, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  (6.0, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">).toDF(&quot;col1&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TabularLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setKernelWidth(0.001)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(0, 1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularLIME">TabularLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.html">TabularLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularLIME.scala">TabularLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="tabularshap">TabularSHAP<a href="#tabularshap" class="hash-link" aria-label="Direct link to TabularSHAP" title="Direct link to TabularSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">data </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;a&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;b&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token operator">*</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;col3&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">data</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = (1 to 100).flatMap(_ =&gt; Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;a&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-5d, &quot;b&quot;, -5d, 0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;a&quot;, 5d, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5d, &quot;b&quot;, 5d, 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )).toDF(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TabularSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TabularSHAP">TabularSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.html">TabularSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TabularSHAP.scala">TabularSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textlime">TextLIME<a href="#textlime" class="hash-link" aria-label="Direct link to TextLIME" title="Direct link to TextLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSamplingFraction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new TextLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setSamplingFraction(0.7)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextLIME">TextLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.html">TextLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextLIME.scala">TextLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textshap">TextSHAP<a href="#textshap" class="hash-link" aria-label="Direct link to TextSHAP" title="Direct link to TextSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prob&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTokensCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = (new ONNXModel())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new TextSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;prob&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTokensCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.TextSHAP">TextSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.html">TextSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/TextSHAP.scala">TextSHAP</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorlime">VectorLIME<a href="#vectorlime" class="hash-link" aria-label="Direct link to VectorLIME" title="Direct link to VectorLIME">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.2729799734928408</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4637273304253777</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.565593782147994</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.541185129673482</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1.9511879801376864</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.495644437589599</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">0.4667847796501322</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.19526424470709836</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lime </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorLIME</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBackgroundData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;weights&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.Rand</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.regression.LinearRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d1 = 3</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val d2 = 1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val coefficients: BDM[Double] = new BDM(d1, d2, Array(1.0, -1.0, 2.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val nRows = 100</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val intercept: Double = math.random()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val x: BDM[Double] = BDM.rand(nRows, d1, Rand.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val y = x * coefficients + intercept</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val xRows = x(*, ::).iterator.toSeq.map(dv =&gt; Vectors.dense(dv.toArray))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    val yRows = y(*, ::).iterator.toSeq.map(dv =&gt; dv(0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    xRows.zip(yRows).toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model: LinearRegressionModel = new LinearRegression().fit(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lime = (new VectorLIME()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(model.getPredictionCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;weights&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorLIME">VectorLIME</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.html">VectorLIME</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorLIME.scala">VectorLIME</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="vectorshap">VectorSHAP<a href="#vectorshap" class="hash-link" aria-label="Direct link to VectorSHAP" title="Direct link to VectorSHAP">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">explainers </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model </span><span class="token operator">=</span><span class="token plain"> ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.explainers._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.linalg.{*, DenseMatrix =&gt; BDM}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import breeze.stats.distributions.RandBasis</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg.Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val randBasis = RandBasis.withSeed(123)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val m: BDM[Double] = BDM.rand[Double](1000, 5, randBasis.gaussian)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val l: BDV[Double] = m(*, ::).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    row =&gt;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      if (row(2) + row(3) &gt; 0.5) 1d else 0d</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val data = m(*, ::).iterator.zip(l.valuesIterator).map {</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    case (f, l) =&gt; (f.toSpark, l)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  }.toSeq.toDF(&quot;features&quot;, &quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFeaturesCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .fit(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val shap = (new VectorSHAP()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;features&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;shapValues&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setBackgroundData(data)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setNumSamples(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setModel(model)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetCol(&quot;probability&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setTargetClasses(Array(1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val infer = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    Tuple1(Vectors.dense(1d, 1d, 1d, 1d, 1d))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ) toDF &quot;features&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val predicted = model.transform(infer)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">shap.transform(predicted).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.explainers.html#module-synapse.ml.explainers.VectorSHAP">VectorSHAP</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.html">VectorSHAP</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/explainers/VectorSHAP.scala">VectorSHAP</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize">Featurize<a href="#featurize" class="hash-link" aria-label="Direct link to Featurize" title="Direct link to Featurize">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dataconversion">DataConversion<a href="#dataconversion" class="hash-link" aria-label="Direct link to DataConversion" title="Direct link to DataConversion">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;7&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;8.0&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">14.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">15.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;16&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;17.456&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">345</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">666</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1234</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">18.91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">20.21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;100&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;200.12345&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;short&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;intstring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doublestring&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">DataConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;byte&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConvertTo</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;boolean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true: Boolean, 1: Byte, 2: Short, 3: Integer, 4: Long, 5.0F, 6.0, &quot;7&quot;, &quot;8.0&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (false, 9: Byte, 10: Short, 11: Integer, 12: Long, 14.5F, 15.5, &quot;16&quot;, &quot;17.456&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (true, -127: Byte, 345: Short, Short.MaxValue + 100, (Int.MaxValue).toLong + 100, 18.91F, 20.21, &quot;100&quot;, &quot;200.12345&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;bool&quot;, &quot;byte&quot;, &quot;short&quot;, &quot;int&quot;, &quot;long&quot;, &quot;float&quot;, &quot;double&quot;, &quot;intstring&quot;, &quot;doublestring&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = (new DataConversion()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setCols(Array(&quot;byte&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConvertTo(&quot;boolean&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.DataConversion">DataConversion</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.html">DataConversion</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/DataConversion.scala">DataConversion</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="indextovalue">IndexToValue<a href="#indextovalue" class="hash-link" aria-label="Direct link to IndexToValue" title="Direct link to IndexToValue">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5.67</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.32534</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;int&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;long&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;double&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bool&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df2 </span><span class="token operator">=</span><span class="token plain"> ValueIndexer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">IndexToValue</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_cat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;string_noncat&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq[(Int, Long, Double, Boolean, String)](</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 24L, 0.32534, true, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, 5L, 5.67, false, &quot;piano&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (-3, 5L, 0.32534, false, &quot;guitar&quot;)).toDF(&quot;int&quot;, &quot;long&quot;, &quot;double&quot;, &quot;bool&quot;, &quot;string&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df2 = new ValueIndexer().setInputCol(&quot;string&quot;).setOutputCol(&quot;string_cat&quot;).fit(df).transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val itv = (new IndexToValue()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;string_cat&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;string_noncat&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">itv.transform(df2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.html#module-synapse.ml.featurize.IndexToValue">IndexToValue</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.html">IndexToValue</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/IndexToValue.scala">IndexToValue</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="featurize-text">Featurize Text<a href="#featurize-text" class="hash-link" aria-label="Direct link to Featurize Text" title="Direct link to Featurize Text">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="multingram">MultiNGram<a href="#multingram" class="hash-link" aria-label="Direct link to MultiNGram" title="Direct link to MultiNGram">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfRaw </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Hi I&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I wish for snow today&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;we Cant go to the park, because of the snow!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1 2 3 4 5 6 7 8 9&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dfTok </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Tokenizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentence&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfRaw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">MultiNGram</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLengths</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;tokens&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;ngrams&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">dfTok</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.Tokenizer</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfRaw = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Hi I&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;I wish for snow today&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;we Cant go to the park, because of the snow!&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, (1 to 10).map(_.toString).mkString(&quot; &quot;)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;sentence&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dfTok = (new Tokenizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;sentence&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .transform(dfRaw))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val mng = (new MultiNGram()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setLengths(Array(1, 3, 4))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;tokens&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;ngrams&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mng.transform(dfTok).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.MultiNGram">MultiNGram</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.html">MultiNGram</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/MultiNGram.scala">MultiNGram</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="pagesplitter">PageSplitter<a href="#pagesplitter" class="hash-link" aria-label="Direct link to PageSplitter" title="Direct link to PageSplitter">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">text </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;s s  s   s     s           s&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd 190872340870271091309831097813097130i3u709781&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">PageSplitter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaximumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMinimumPageLength</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;pages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.featurize.text._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;words words  words     wornssaa ehewjkdiw weijnsikjn xnh&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;s s  s   s     s           s&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hsjbhjhnskjhndwjnbvckjbnwkjwenbvfkjhbnwevkjhbnwejhkbnvjkhnbndjkbnd &quot; +</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      &quot;190872340870271091309831097813097130i3u709781&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    null</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ps = (new PageSplitter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;text&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMaximumPageLength(20)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setMinimumPageLength(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;pages&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ps.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.featurize.text.html#module-synapse.ml.featurize.text.PageSplitter">PageSplitter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.html">PageSplitter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/featurize/text/PageSplitter.scala">PageSplitter</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="image">Image<a href="#image" class="hash-link" aria-label="Direct link to Image" title="Direct link to Image">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollimage">UnrollImage<a href="#unrollimage" class="hash-link" aria-label="Direct link to UnrollImage" title="Direct link to UnrollImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> azure</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">storage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">blob </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># rit = (ResizeImageTransformer()</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setOutputCol(&quot;out&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setHeight(15)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .setWidth(10))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># preprocessed = rit.transform(images)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://datasets@mmlspark.blob.core.windows.net/LIME/greyscale.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rit = (new ResizeImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setHeight(15)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setWidth(10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val preprocessed = rit.transform(images)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(rit.getOutputCol)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;final&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll.transform(preprocessed).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollImage">UnrollImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.html">UnrollImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollImage.scala">UnrollImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unrollbinaryimage">UnrollBinaryImage<a href="#unrollbinaryimage" class="hash-link" aria-label="Direct link to UnrollBinaryImage" title="Direct link to UnrollBinaryImage">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unroll </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnrollBinaryImage</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;input_col&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;final&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import spark.implicits._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unroll = (new UnrollBinaryImage()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;input_col&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;final&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.image.html#module-synapse.ml.image.UnrollBinaryImage">UnrollBinaryImage</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.html">UnrollBinaryImage</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/UnrollBinaryImage.scala">UnrollBinaryImage</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="superpixeltransformer">SuperpixelTransformer<a href="#superpixeltransformer" class="hash-link" aria-label="Direct link to SuperpixelTransformer" title="Direct link to SuperpixelTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">image </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">spt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SuperpixelTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.image._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val spt = (new SuperpixelTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;images&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.lime.html#module-synapse.ml.image.SuperpixelTransformer">SuperpixelTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.html">SuperpixelTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/image/SuperpixelTransformer.scala">SuperpixelTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="io">IO<a href="#io" class="hash-link" aria-label="Direct link to IO" title="Direct link to IO">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="httptransformer">HTTPTransformer<a href="#httptransformer" class="hash-link" aria-label="Direct link to HTTPTransformer" title="Direct link to HTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> col</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> requests </span><span class="token keyword">import</span><span class="token plain"> Request</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">return</span><span class="token plain"> Request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;GET&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;http://api.worldbank.org/v2/country/{}?format=json&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token builtin" style="color:rgb(174, 129, 255)">format</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">country</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;br&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;usa&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">withColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> http_udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">world_bank_request</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">HTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;request&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;response&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ht</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ht = (new HTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setConcurrency(3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;request&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;response&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.HTTPTransformer">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/HTTPTransformer.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="simplehttptransformer">SimpleHTTPTransformer<a href="#simplehttptransformer" class="hash-link" aria-label="Direct link to SimpleHTTPTransformer" title="Direct link to SimpleHTTPTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sht </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">SimpleHTTPTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;blah&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;results&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setConcurrency</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sht = (new SimpleHTTPTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputParser(new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            .setDataType(new StructType().add(&quot;blah&quot;, StringType)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setUrl(&quot;PUT_YOUR_URL&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setOutputCol(&quot;results&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setConcurrency(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.SimpleHTTPTransformer">SimpleHTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.html">SimpleHTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/SimpleHTTPTransformer.scala">SimpleHTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsoninputparser">JSONInputParser<a href="#jsoninputparser" class="hash-link" aria-label="Direct link to JSONInputParser" title="Direct link to JSONInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonIP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUrl</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_URL&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonIP = (new JSONInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUrl(&quot;PUT_YOUR_URL&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONInputParser">JSONInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.html">JSONInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONInputParser.scala">JSONInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="jsonoutputparser">JSONOutputParser<a href="#jsonoutputparser" class="hash-link" aria-label="Direct link to JSONOutputParser" title="Direct link to JSONOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">jsonOP </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">JSONOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDataType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">add</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;parsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val jsonOP = (new JSONOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setDataType(new StructType().add(&quot;foo&quot;, StringType))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;parsedOutput&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.JSONOutputParser">JSONOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.html">JSONOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/JSONOutputParser.scala">JSONOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stringoutputparser">StringOutputParser<a href="#stringoutputparser" class="hash-link" aria-label="Direct link to StringOutputParser" title="Direct link to StringOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">StringOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sop = (new StringOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.StringOutputParser">StringOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.html">StringOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/StringOutputParser.scala">StringOutputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="custominputparser">CustomInputParser<a href="#custominputparser" class="hash-link" aria-label="Direct link to CustomInputParser" title="Direct link to CustomInputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cip </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomInputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;data&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cip = (new CustomInputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;data&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF({ x: Int =&gt; new HttpPost(s&quot;http://$x&quot;) }))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomInputParser">CustomInputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.html">CustomInputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomInputParser.scala">CustomInputParser</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="customoutputparser">CustomOutputParser<a href="#customoutputparser" class="hash-link" aria-label="Direct link to CustomOutputParser" title="Direct link to CustomOutputParser">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">io</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">http </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cop </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">CustomOutputParser</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;unparsedOutput&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.io.http._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cop = (new CustomOutputParser()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;unparsedOutput&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.io.http.html#module-synapse.ml.io.http.CustomOutputParser">CustomOutputParser</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.html">CustomOutputParser</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/io/http/CustomOutputParser.scala">CustomOutputParser</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="stages">Stages<a href="#stages" class="hash-link" aria-label="Direct link to Stages" title="Direct link to Stages">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cacher">Cacher<a href="#cacher" class="hash-link" aria-label="Direct link to Cacher" title="Direct link to Cacher">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher </span><span class="token operator">=</span><span class="token plain"> Cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cacher = new Cacher()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cacher.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Cacher">HTTPTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Cacher.html">HTTPTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Cacher.scala">HTTPTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dropcolumns">DropColumns<a href="#dropcolumns" class="hash-link" aria-label="Direct link to DropColumns" title="Direct link to DropColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc </span><span class="token operator">=</span><span class="token plain"> DropColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dc = new DropColumns().setCols(Array())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DropColumns">DropColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.html">DropColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/DropColumns.scala">DropColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="ensemblebykey">EnsembleByKey<a href="#ensemblebykey" class="hash-link" aria-label="Direct link to EnsembleByKey" title="Direct link to EnsembleByKey">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">feature </span><span class="token keyword">import</span><span class="token plain"> VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bar&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token operator">-</span><span class="token number" style="color:rgb(174, 129, 255)">3.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">va </span><span class="token operator">=</span><span class="token plain"> VectorAssembler</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">scoreDF2 </span><span class="token operator">=</span><span class="token plain"> va</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk </span><span class="token operator">=</span><span class="token plain"> EnsembleByKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setKeys</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;score1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">scoreDF2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.VectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, &quot;foo&quot;, 1.0, .1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 4.0, -2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, &quot;bar&quot;, 0.0, -3.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label1&quot;, &quot;label2&quot;, &quot;score1&quot;, &quot;score2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val va = new VectorAssembler().setInputCols(Array(&quot;score1&quot;, &quot;score2&quot;)).setOutputCol(&quot;v1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoreDF2 = va.transform(scoreDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val ebk = new EnsembleByKey().setKey(&quot;label1&quot;).setCol(&quot;score1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ebk.transform(scoreDF2).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.EnsembleByKey">EnsembleByKey</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.html">EnsembleByKey</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/EnsembleByKey.scala">EnsembleByKey</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="explode">Explode<a href="#explode" class="hash-link" aria-label="Direct link to Explode" title="Direct link to Explode">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode </span><span class="token operator">=</span><span class="token plain"> Explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;exploded&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, Seq(&quot;guitars&quot;, &quot;drums&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, Seq(&quot;piano&quot;)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, Seq()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;numbers&quot;, &quot;words&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val explode = new Explode().setInputCol(&quot;words&quot;).setOutputCol(&quot;exploded&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">explode.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Explode">Explode</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Explode.html">Explode</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Explode.scala">Explode</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lambda">Lambda<a href="#lambda" class="hash-link" aria-label="Direct link to Lambda" title="Direct link to Lambda">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">select</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">def</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token keyword">return</span><span class="token plain"> StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain">schema</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">l </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Lambda</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTransformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transformSchemaFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.types.{StringType, StructType}</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val lambda = (new Lambda()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransform(df =&gt; df.select(&quot;numbers&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setTransformSchema(schema =&gt; new StructType(Array(schema(&quot;numbers&quot;)))))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">lambda.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Lambda">Lambda</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Lambda.html">Lambda</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Lambda.scala">Lambda</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="dynamicminibatchtransformer">DynamicMiniBatchTransformer<a href="#dynamicminibatchtransformer" class="hash-link" aria-label="Direct link to DynamicMiniBatchTransformer" title="Direct link to DynamicMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> StructType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dmbt = new DynamicMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dmbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.DynamicMiniBatchTransformer">DynamicMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/DynamicMiniBatchTransformer.html">DynamicMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">DynamicMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="fixedminibatchtransformer">FixedMiniBatchTransformer<a href="#fixedminibatchtransformer" class="hash-link" aria-label="Direct link to FixedMiniBatchTransformer" title="Direct link to FixedMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fmbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FixedMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBuffered</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fmbt = (new FixedMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBuffered(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setBatchSize(3))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FixedMiniBatchTransformer">FixedMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FixedMiniBatchTransformer.html">FixedMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FixedMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="timeintervalminibatchtransformer">TimeIntervalMiniBatchTransformer<a href="#timeintervalminibatchtransformer" class="hash-link" aria-label="Direct link to TimeIntervalMiniBatchTransformer" title="Direct link to TimeIntervalMiniBatchTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TimeIntervalMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMillisToWait</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMaxBatchSize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val timbt = (new TimeIntervalMiniBatchTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMillisToWait(1000)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        .setMaxBatchSize(30))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">timbt.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TimeIntervalMiniBatchTransformer">TimeIntervalMiniBatchTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TimeIntervalMiniBatchTransformer.html">TimeIntervalMiniBatchTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">TimeIntervalMiniBatchTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="flattenbatch">FlattenBatch<a href="#flattenbatch" class="hash-link" aria-label="Direct link to FlattenBatch" title="Direct link to FlattenBatch">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">_</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">transDF </span><span class="token operator">=</span><span class="token plain"> DynamicMiniBatchTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb </span><span class="token operator">=</span><span class="token plain"> FlattenBatch</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">transDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (1 until 11).map(x =&gt; (x, &quot;foo&quot;)).toDF(&quot;in1&quot;, &quot;in2&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val transDF = new DynamicMiniBatchTransformer().transform(df)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val fb = new FlattenBatch()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">fb.transform(transDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.FlattenBatch">FlattenBatch</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/FlattenBatch.html">FlattenBatch</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/MiniBatchTransformer.scala">FlattenBatch</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="renamecolumn">RenameColumn<a href="#renamecolumn" class="hash-link" aria-label="Direct link to RenameColumn" title="Direct link to RenameColumn">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc </span><span class="token operator">=</span><span class="token plain"> RenameColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rc = new RenameColumn().setInputCol(&quot;words&quot;).setOutputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">rc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.RenameColumn">RenameColumn</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.html">RenameColumn</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/RenameColumn.scala">RenameColumn</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="repartition">Repartition<a href="#repartition" class="hash-link" aria-label="Direct link to Repartition" title="Direct link to Repartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">11</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition </span><span class="token operator">=</span><span class="token plain"> Repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (4, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (5, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (6, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (7, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (8, &quot;bass&quot;, &quot;cymbals&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (9, &quot;guitars&quot;, &quot;drums&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (10, &quot;piano&quot;, &quot;trumpet&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (11, &quot;bass&quot;, &quot;cymbals&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;numbers&quot;, &quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val repartition = new Repartition().setN(1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">repartition.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.Repartition">Repartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/Repartition.html">Repartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/Repartition.scala">Repartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="selectcolumns">SelectColumns<a href="#selectcolumns" class="hash-link" aria-label="Direct link to SelectColumns" title="Direct link to SelectColumns">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc </span><span class="token operator">=</span><span class="token plain"> SelectColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sc = new SelectColumns().setCols(Array(&quot;words&quot;, &quot;more&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sc.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SelectColumns">SelectColumns</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.html">SelectColumns</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SelectColumns.scala">SelectColumns</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="stratifiedrepartition">StratifiedRepartition<a href="#stratifiedrepartition" class="hash-link" aria-label="Direct link to StratifiedRepartition" title="Direct link to StratifiedRepartition">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Blue&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Red&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Green&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Purple&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Orange&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Indigo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Violet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Black&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;White&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Gray&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Yellow&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Cerulean&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;colors&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;const&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr </span><span class="token operator">=</span><span class="token plain"> StratifiedRepartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMode</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;equal&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Blue&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Red&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0, &quot;Green&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Purple&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Orange&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1, &quot;Indigo&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Violet&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;Black&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2, &quot;White&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Gray&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Yellow&quot;, 2),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3, &quot;Cerulean&quot;, 2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  ).toDF(&quot;values&quot;, &quot;colors&quot;, &quot;const&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val sr = new StratifiedRepartition().setLabelCol(&quot;values&quot;).setMode(&quot;equal&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">sr.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.StratifiedRepartition">StratifiedRepartition</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.html">StratifiedRepartition</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/StratifiedRepartition.scala">StratifiedRepartition</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="summarizedata">SummarizeData<a href="#summarizedata" class="hash-link" aria-label="Direct link to SummarizeData" title="Direct link to SummarizeData">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary </span><span class="token operator">=</span><span class="token plain"> SummarizeData</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val summary = new SummarizeData()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">summary.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.SummarizeData">SummarizeData</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.html">SummarizeData</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/SummarizeData.scala">SummarizeData</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="textpreprocessor">TextPreprocessor<a href="#textpreprocessor" class="hash-link" aria-label="Direct link to TextPreprocessor" title="Direct link to TextPreprocessor">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The happy sad boy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy drank sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The hater sad doy aABc0123456789Zz_&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">testMap </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;happy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hater&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sad doy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNormFunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lowerCase&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testMap</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The happy sad boy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy drank sap&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;foo&quot;, ),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;The hater sad doy aABc0123456789Zz_&quot;, ))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val testMap = Map[String, String] (</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;happy&quot;   -&gt; &quot;sad&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;hater&quot;   -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad&quot;     -&gt; &quot;sap&quot;,</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    &quot;sad doy&quot; -&gt; &quot;sap&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  )</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val textPreprocessor = (new TextPreprocessor()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNormFunc(&quot;lowerCase&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMap(testMap)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">textPreprocessor.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.TextPreprocessor">TextPreprocessor</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.html">TextPreprocessor</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/TextPreprocessor.scala">TextPreprocessor</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="udftransformer">UDFTransformer<a href="#udftransformer" class="hash-link" aria-label="Direct link to UDFTransformer" title="Direct link to UDFTransformer">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">functions </span><span class="token keyword">import</span><span class="token plain"> udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;guitars&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;drums&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;piano&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;trumpet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;bass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;cymbals&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;doubles&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;more&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;longs&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;booleans&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">stringToIntegerUDF </span><span class="token operator">=</span><span class="token plain"> udf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">lambda</span><span class="token plain"> x</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UDFTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">stringToIntegerUDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;numbers&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.sql.functions.udf</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (0, 0.toDouble, &quot;guitars&quot;, &quot;drums&quot;, 1.toLong, true),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (1, 1.toDouble, &quot;piano&quot;, &quot;trumpet&quot;, 2.toLong, false),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (2, 2.toDouble, &quot;bass&quot;, &quot;cymbals&quot;, 3.toLong, true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;numbers&quot;, &quot;doubles&quot;, &quot;words&quot;, &quot;more&quot;, &quot;longs&quot;, &quot;booleans&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val stringToIntegerUDF = udf((_: String) =&gt; 1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val udfTransformer = (new UDFTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setUDF(stringToIntegerUDF)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;numbers&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">udfTransformer.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UDFTransformer">UDFTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.html">UDFTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UDFTransformer.scala">UDFTransformer</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unicodenormalize">UnicodeNormalize<a href="#unicodenormalize" class="hash-link" aria-label="Direct link to UnicodeNormalize" title="Direct link to UnicodeNormalize">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">stages </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Schön&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Scho\u0308n&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">None</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;dummy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">UnicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setForm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;NFC&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;words1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;norm1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.stages._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Schön&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (&quot;Scho\u0308n&quot;, 1),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (null, 1))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;words1&quot;, &quot;dummy&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val unicodeNormalize = (new UnicodeNormalize()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setForm(&quot;NFC&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCol(&quot;words1&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;norm1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">unicodeNormalize.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.stages.html#module-synapse.ml.stages.UnicodeNormalize">UnicodeNormalize</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.html">UnicodeNormalize</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/stages/UnicodeNormalize.scala">UnicodeNormalize</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="train">Train<a href="#train" class="hash-link" aria-label="Direct link to Train" title="Direct link to Train">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computemodelstatistics">ComputeModelStatistics<a href="#computemodelstatistics" class="hash-link" aria-label="Direct link to ComputeModelStatistics" title="Direct link to ComputeModelStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> numpy </span><span class="token keyword">import</span><span class="token plain"> random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> random</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">rand</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token keyword">for</span><span class="token plain"> _ </span><span class="token keyword">in</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">range</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2048</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import scala.util.Random</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val rand = new Random(1337)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq.fill(2048)(rand.nextDouble())</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .zip(Seq.fill(2048)(rand.nextDouble()))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .toDF(&quot;label&quot;, &quot;prediction&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cms = (new ComputeModelStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cms.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputeModelStatistics">ComputeModelStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.html">ComputeModelStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputeModelStatistics.scala">ComputeModelStatistics</a></td></tr></tbody></table><h3 class="anchor anchorWithStickyNavbar_LWe7" id="computeperinstancestatistics">ComputePerInstanceStatistics<a href="#computeperinstancestatistics" class="hash-link" aria-label="Direct link to ComputePerInstanceStatistics" title="Direct link to ComputePerInstanceStatistics">​</a></h3><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ComputePerInstanceStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredLabelsCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoredLabelsCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoresCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegScoresCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setScoredProbabilitiesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LogRegProbCol&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setEvaluationMetric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.train._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.classification.LogisticRegression</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.feature.FastVectorAssembler</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val logisticRegression = (new LogisticRegression()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRegParam(0.3)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setElasticNetParam(0.8)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setMaxIter(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPredictionCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setRawPredictionCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setProbabilityCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setFeaturesCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val dataset = spark.createDataFrame(Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 2, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 4, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 5, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 1, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 3, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (0.0, 0, 0.50, 0.60, 0.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (1.0, 2, 0.40, 0.50, 1.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (2.0, 3, 0.78, 0.99, 2.0),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    (3.0, 4, 0.12, 0.34, 3.0)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .toDF(&quot;label&quot;, &quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;, &quot;prediction&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembler = (new FastVectorAssembler()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setInputCols(Array(&quot;col1&quot;, &quot;col2&quot;, &quot;col3&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val assembledDataset = assembler.transform(dataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model = logisticRegression.fit(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val scoredData = model.transform(assembledDataset)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val cps = (new ComputePerInstanceStatistics()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelCol(&quot;label&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredLabelsCol(&quot;LogRegScoredLabelsCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoresCol(&quot;LogRegScoresCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setScoredProbabilitiesCol(&quot;LogRegProbCol&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setEvaluationMetric(&quot;classification&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cps.transform(scoredData).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.train.html#module-synapse.ml.train.ComputePerInstanceStatistics">ComputePerInstanceStatistics</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.html">ComputePerInstanceStatistics</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/core/src/main/scala/com/microsoft/azure/synapse/ml/train/ComputePerInstanceStatistics.scala">ComputePerInstanceStatistics</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#explainers" class="table-of-contents__link toc-highlight">Explainers</a><ul><li><a href="#imagelime" class="table-of-contents__link toc-highlight">ImageLIME</a></li><li><a href="#imageshap" class="table-of-contents__link toc-highlight">ImageSHAP</a></li><li><a href="#tabularlime" class="table-of-contents__link toc-highlight">TabularLIME</a></li><li><a href="#tabularshap" class="table-of-contents__link toc-highlight">TabularSHAP</a></li><li><a href="#textlime" class="table-of-contents__link toc-highlight">TextLIME</a></li><li><a href="#textshap" class="table-of-contents__link toc-highlight">TextSHAP</a></li><li><a href="#vectorlime" class="table-of-contents__link toc-highlight">VectorLIME</a></li><li><a href="#vectorshap" class="table-of-contents__link toc-highlight">VectorSHAP</a></li></ul></li><li><a href="#featurize" class="table-of-contents__link toc-highlight">Featurize</a><ul><li><a href="#dataconversion" class="table-of-contents__link toc-highlight">DataConversion</a></li><li><a href="#indextovalue" class="table-of-contents__link toc-highlight">IndexToValue</a></li></ul></li><li><a href="#featurize-text" class="table-of-contents__link toc-highlight">Featurize Text</a><ul><li><a href="#multingram" class="table-of-contents__link toc-highlight">MultiNGram</a></li><li><a href="#pagesplitter" class="table-of-contents__link toc-highlight">PageSplitter</a></li></ul></li><li><a href="#image" class="table-of-contents__link toc-highlight">Image</a><ul><li><a href="#unrollimage" class="table-of-contents__link toc-highlight">UnrollImage</a></li><li><a href="#unrollbinaryimage" class="table-of-contents__link toc-highlight">UnrollBinaryImage</a></li><li><a href="#superpixeltransformer" class="table-of-contents__link toc-highlight">SuperpixelTransformer</a></li></ul></li><li><a href="#io" class="table-of-contents__link toc-highlight">IO</a><ul><li><a href="#httptransformer" class="table-of-contents__link toc-highlight">HTTPTransformer</a></li><li><a href="#simplehttptransformer" class="table-of-contents__link toc-highlight">SimpleHTTPTransformer</a></li><li><a href="#jsoninputparser" class="table-of-contents__link toc-highlight">JSONInputParser</a></li><li><a href="#jsonoutputparser" class="table-of-contents__link toc-highlight">JSONOutputParser</a></li><li><a href="#stringoutputparser" class="table-of-contents__link toc-highlight">StringOutputParser</a></li><li><a href="#custominputparser" class="table-of-contents__link toc-highlight">CustomInputParser</a></li><li><a href="#customoutputparser" class="table-of-contents__link toc-highlight">CustomOutputParser</a></li></ul></li><li><a href="#stages" class="table-of-contents__link toc-highlight">Stages</a><ul><li><a href="#cacher" class="table-of-contents__link toc-highlight">Cacher</a></li><li><a href="#dropcolumns" class="table-of-contents__link toc-highlight">DropColumns</a></li><li><a href="#ensemblebykey" class="table-of-contents__link toc-highlight">EnsembleByKey</a></li><li><a href="#explode" class="table-of-contents__link toc-highlight">Explode</a></li><li><a href="#lambda" class="table-of-contents__link toc-highlight">Lambda</a></li><li><a href="#dynamicminibatchtransformer" class="table-of-contents__link toc-highlight">DynamicMiniBatchTransformer</a></li><li><a href="#fixedminibatchtransformer" class="table-of-contents__link toc-highlight">FixedMiniBatchTransformer</a></li><li><a href="#timeintervalminibatchtransformer" class="table-of-contents__link toc-highlight">TimeIntervalMiniBatchTransformer</a></li><li><a href="#flattenbatch" class="table-of-contents__link toc-highlight">FlattenBatch</a></li><li><a href="#renamecolumn" class="table-of-contents__link toc-highlight">RenameColumn</a></li><li><a href="#repartition" class="table-of-contents__link toc-highlight">Repartition</a></li><li><a href="#selectcolumns" class="table-of-contents__link toc-highlight">SelectColumns</a></li><li><a href="#stratifiedrepartition" class="table-of-contents__link toc-highlight">StratifiedRepartition</a></li><li><a href="#summarizedata" class="table-of-contents__link toc-highlight">SummarizeData</a></li><li><a href="#textpreprocessor" class="table-of-contents__link toc-highlight">TextPreprocessor</a></li><li><a href="#udftransformer" class="table-of-contents__link toc-highlight">UDFTransformer</a></li><li><a href="#unicodenormalize" class="table-of-contents__link toc-highlight">UnicodeNormalize</a></li></ul></li><li><a href="#train" class="table-of-contents__link toc-highlight">Train</a><ul><li><a href="#computemodelstatistics" class="table-of-contents__link toc-highlight">ComputeModelStatistics</a></li><li><a href="#computeperinstancestatistics" class="table-of-contents__link toc-highlight">ComputePerInstanceStatistics</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/transformers/transformers_deep_learning/index.html b/docs/next/Quick Examples/transformers/transformers_deep_learning/index.html
index 80ecb52889..3d01bad883 100644
--- a/docs/next/Quick Examples/transformers/transformers_deep_learning/index.html	
+++ b/docs/next/Quick Examples/transformers/transformers_deep_learning/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_deep_learning/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_deep_learning/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_deep_learning/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_deep_learning/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_deep_learning/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/transformers/transformers_deep_learning/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_deep_learning/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Deep Learning</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="onnxmodel">ONNXModel<a href="#onnxmodel" class="hash-link" aria-label="Direct link to ONNXModel" title="Direct link to ONNXModel">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-py codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-py codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">onnx </span><span class="token keyword">import</span><span class="token plain"> ONNXModel</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">model_path </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;PUT_YOUR_MODEL_PATH&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">onnx_ml </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ONNXModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModelLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model_path</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeedDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;float_input&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFetchDict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawProbability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;output_probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.onnx._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val model_path = &quot;PUT_YOUR_MODEL_PATH&quot;</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val onnx_ml = (new ONNXModel()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setModelLocation(model_path)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFeedDict(Map(&quot;float_input&quot; -&gt; &quot;features&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                  .setFetchDict(Map(&quot;prediction&quot; -&gt; &quot;output_label&quot;, &quot;rawProbability&quot; -&gt; &quot;output_probability&quot;)))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.onnx.html#module-synapse.ml.onnx.ONNXModel">ONNXModel</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.html">ONNXModel</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/deep-learning/src/main/scala/com/microsoft/azure/synapse/ml/onnx/ONNXModel.scala">ONNXModel</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#onnxmodel" class="table-of-contents__link toc-highlight">ONNXModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/transformers/transformers_opencv/index.html b/docs/next/Quick Examples/transformers/transformers_opencv/index.html
index 2dd6a3307c..391aa875c1 100644
--- a/docs/next/Quick Examples/transformers/transformers_opencv/index.html	
+++ b/docs/next/Quick Examples/transformers/transformers_opencv/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_opencv/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_opencv/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_opencv/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_opencv/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_opencv/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/transformers/transformers_opencv/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_opencv/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>OpenCV</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagetransformer">ImageTransformer<a href="#imagetransformer" class="hash-link" aria-label="Direct link to ImageTransformer" title="Direct link to ImageTransformer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">types </span><span class="token keyword">import</span><span class="token plain"> FloatType</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">inputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> outputCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">height</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> width</span><span class="token operator">=</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTensorElementType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">FloatType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val it = (new ImageTransformer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;out&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .resize(height = 15, width = 10))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageTransformer">ImageTransformer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.html">ImageTransformer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageTransformer.scala">ImageTransformer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="imagesetaugmenter">ImageSetAugmenter<a href="#imagesetaugmenter" class="hash-link" aria-label="Direct link to ImageSetAugmenter" title="Direct link to ImageSetAugmenter">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># images = (spark.read.format(&quot;image&quot;)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .option(&quot;dropInvalid&quot;, True)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#         .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageSetAugmenter</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;image&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;augmented&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipLeftRight</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFlipUpDown</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)"># it.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.opencv._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val images = (spark.read.format(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .option(&quot;dropInvalid&quot;, true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .load(&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/explainers/images/david-lusvardi-dWcUncxocQY-unsplash.jpg&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val isa = (new ImageSetAugmenter()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCol(&quot;image&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;augmented&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipLeftRight(true)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setFlipUpDown(true))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">isa.transform(images).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.opencv.html#module-synapse.ml.opencv.ImageSetAugmenter">ImageSetAugmenter</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.html">ImageSetAugmenter</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/opencv/src/main/scala/com/microsoft/azure/synapse/ml/opencv/ImageSetAugmenter.scala">ImageSetAugmenter</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#imagetransformer" class="table-of-contents__link toc-highlight">ImageTransformer</a></li><li><a href="#imagesetaugmenter" class="table-of-contents__link toc-highlight">ImageSetAugmenter</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Quick Examples/transformers/transformers_vw/index.html b/docs/next/Quick Examples/transformers/transformers_vw/index.html
index dfc30e08d9..bef14ab4b1 100644
--- a/docs/next/Quick Examples/transformers/transformers_vw/index.html	
+++ b/docs/next/Quick Examples/transformers/transformers_vw/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Quick Examples/transformers/transformers_vw/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Quick Examples/transformers/transformers_vw/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Quick Examples/transformers/transformers_vw/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Quick Examples/transformers/transformers_vw/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Quick Examples/transformers/transformers_vw/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Quick Examples/transformers/transformers_vw/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Quick Examples/transformers/transformers_vw/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><h1>Vowpal Wabbit</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vectorzipper">VectorZipper<a href="#vectorzipper" class="hash-link" aria-label="Direct link to VectorZipper" title="Direct link to VectorZipper">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">.NET</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = (Seq(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      (&quot;action1_f&quot;, &quot;action2_f&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    ).toDF(&quot;action1&quot;, &quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionOneFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_one&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val actionTwoFeaturizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;action2&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val seqDF = actionTwoFeaturizer.transform(actionOneFeaturizer.transform(df))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vectorZipper = (new VectorZipper()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;sequence_one&quot;, &quot;sequence_two&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vectorZipper.transform(seqDF).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Vw</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Example&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">CreateDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">GenericRow</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">GenericRow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">object</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2_f&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">List</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&lt;</span><span class="token constructor-invocation class-name">StructField</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">&gt;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StructField</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">StringType</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionOneFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> actionTwoFeaturizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;action2&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> seqDF </span><span class="token operator">=</span><span class="token plain"> actionTwoFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">actionOneFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> vectorZipper </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">VectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_one&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sequence_two&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            vectorZipper</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">seqDF</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VectorZipper">VectorZipper</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.html">VectorZipper</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VectorZipper.scala">VectorZipper</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitclassifier">VowpalWabbitClassifier<a href="#vowpalwabbitclassifier" class="hash-link" aria-label="Direct link to VowpalWabbitClassifier" title="Direct link to VowpalWabbitClassifier">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">vw </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPowerT</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelConversion</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val vw = (new VowpalWabbitClassifier()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(10)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLearningRate(3.1)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPowerT(0)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setLabelConversion(false))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitClassifier">VowpalWabbitClassifier</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.html">VowpalWabbitClassifier</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitClassifier.scala">VowpalWabbitClassifier</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitfeaturizer">VowpalWabbitFeaturizer<a href="#vowpalwabbitfeaturizer" class="hash-link" aria-label="Direct link to VowpalWabbitFeaturizer" title="Direct link to VowpalWabbitFeaturizer">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">featurizer </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitFeaturizer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setStringSplitInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;in&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPreserveOrderNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumBits</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">18</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setPrefixStringsWithColumnName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">False</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val featurizer = (new VowpalWabbitFeaturizer()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setStringSplitInputCols(Array(&quot;in&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPreserveOrderNumBits(2)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setNumBits(18)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setPrefixStringsWithColumnName(false)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">      .setOutputCol(&quot;features&quot;))</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitFeaturizer">VowpalWabbitFeaturizer</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.html">VowpalWabbitFeaturizer</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitFeaturizer.scala">VowpalWabbitFeaturizer</a></td></tr></tbody></table><h2 class="anchor anchorWithStickyNavbar_LWe7" id="vowpalwabbitinteractions">VowpalWabbitInteractions<a href="#vowpalwabbitinteractions" class="hash-link" aria-label="Direct link to VowpalWabbitInteractions" title="Direct link to VowpalWabbitInteractions">​</a></h2><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Scala</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6"><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">vw </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">VowpalWabbitInteractions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;v1&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;out&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="language-scala codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-scala codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">import com.microsoft.azure.synapse.ml.vw._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">import org.apache.spark.ml.linalg._</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">case class Data(v1: Vector, v2: Vector, v3: Vector)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val df = spark.createDataFrame(Seq(Data(</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.dense(Array(1.0, 2.0, 3.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(8, Array(5), Array(4.0)),</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">  Vectors.sparse(11, Array(8, 9), Array(7.0, 8.0))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">)))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">val interactions = (new VowpalWabbitInteractions()</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setInputCols(Array(&quot;v1&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    .setOutputCol(&quot;out&quot;))</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">interactions.transform(df).show()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></div></div><table><tbody><tr><td><strong>Python API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/synapse.ml.vw.html#module-synapse.ml.vw.VowpalWabbitInteractions">VowpalWabbitInteractions</a></td><td><strong>Scala API: </strong><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.html">VowpalWabbitInteractions</a></td><td><strong>Source: </strong><a href="https://github.com/microsoft/SynapseML/blob/master/vw/src/main/scala/com/microsoft/azure/synapse/ml/vw/VowpalWabbitInteractions.scala">VowpalWabbitInteractions</a></td></tr></tbody></table></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#vectorzipper" class="table-of-contents__link toc-highlight">VectorZipper</a></li><li><a href="#vowpalwabbitclassifier" class="table-of-contents__link toc-highlight">VowpalWabbitClassifier</a></li><li><a href="#vowpalwabbitfeaturizer" class="table-of-contents__link toc-highlight">VowpalWabbitFeaturizer</a></li><li><a href="#vowpalwabbitinteractions" class="table-of-contents__link toc-highlight">VowpalWabbitInteractions</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/Contributor Guide/index.html b/docs/next/Reference/Contributor Guide/index.html
index c97064c098..d233038cbf 100644
--- a/docs/next/Reference/Contributor Guide/index.html	
+++ b/docs/next/Reference/Contributor Guide/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -42,7 +42,7 @@
 out your features sooner by using build artifacts for the version that has
 your changes merged in (such versions end with a <code>.devN</code>).</li></ul><p>If in doubt about how to do something, see how it was done in existing code or
 pull requests, and don&#x27;t hesitate to ask.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Deploy Models/Quickstart - Deploying a Classifier/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Deploying a Classifier</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Developer Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#interested-in-contributing-to-synapseml--were-excited-to-work-with-you" class="table-of-contents__link toc-highlight">Interested in contributing to SynapseML?  We&#39;re excited to work with you.</a><ul><li><a href="#you-can-contribute-in-many-ways" class="table-of-contents__link toc-highlight">You can contribute in many ways:</a></li><li><a href="#how-to-contribute" class="table-of-contents__link toc-highlight">How to contribute?</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/Developer Setup/index.html b/docs/next/Reference/Developer Setup/index.html
index 49b0f921a9..1d85910826 100644
--- a/docs/next/Reference/Developer Setup/index.html	
+++ b/docs/next/Reference/Developer Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -30,7 +30,7 @@
 added, reach out to <code>synapseml-support@microsoft.com</code></p><h1>SBT Command Guide</h1><h2 class="anchor anchorWithStickyNavbar_LWe7" id="scala-build-commands">Scala build commands<a href="#scala-build-commands" class="hash-link" aria-label="Direct link to Scala build commands" title="Direct link to Scala build commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="compile-testcompile-and-itcompile"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code><a href="#compile-testcompile-and-itcompile" class="hash-link" aria-label="Direct link to compile-testcompile-and-itcompile" title="Direct link to compile-testcompile-and-itcompile">​</a></h3><p>Compiles the main, test, and integration test classes respectively</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="test"><code>test</code><a href="#test" class="hash-link" aria-label="Direct link to test" title="Direct link to test">​</a></h3><p>Runs all synapsemltests</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="scalastyle"><code>scalastyle</code><a href="#scalastyle" class="hash-link" aria-label="Direct link to scalastyle" title="Direct link to scalastyle">​</a></h3><p>Runs scalastyle check on main</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testscalastyle"><code>test:scalastyle</code><a href="#testscalastyle" class="hash-link" aria-label="Direct link to testscalastyle" title="Direct link to testscalastyle">​</a></h3><p>Runs scalastyle check on test</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="unidoc"><code>unidoc</code><a href="#unidoc" class="hash-link" aria-label="Direct link to unidoc" title="Direct link to unidoc">​</a></h3><p>Generates documentation for scala sources</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="python-commands">Python Commands<a href="#python-commands" class="hash-link" aria-label="Direct link to Python Commands" title="Direct link to Python Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="createcondaenv"><code>createCondaEnv</code><a href="#createcondaenv" class="hash-link" aria-label="Direct link to createcondaenv" title="Direct link to createcondaenv">​</a></h3><p>Creates a conda environment <code>synapseml</code> from <code>environment.yml</code> if it doesn&#x27;t already exist.
 This env is used for python testing.
 <strong>Activate this env before using python build commands.</strong></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="cleancondaenv"><code>cleanCondaEnv</code><a href="#cleancondaenv" class="hash-link" aria-label="Direct link to cleancondaenv" title="Direct link to cleancondaenv">​</a></h3><p>Removes <code>synapseml</code> conda env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="packagepython"><code>packagePython</code><a href="#packagepython" class="hash-link" aria-label="Direct link to packagepython" title="Direct link to packagepython">​</a></h3><p>Compiles scala, runs python generation scripts, and creates a wheel</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="generatepythondoc"><code>generatePythonDoc</code><a href="#generatepythondoc" class="hash-link" aria-label="Direct link to generatepythondoc" title="Direct link to generatepythondoc">​</a></h3><p>Generates documentation for generated python code</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="installpippackage"><code>installPipPackage</code><a href="#installpippackage" class="hash-link" aria-label="Direct link to installpippackage" title="Direct link to installpippackage">​</a></h3><p>Installs generated python wheel into existing env</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="testpython"><code>testPython</code><a href="#testpython" class="hash-link" aria-label="Direct link to testpython" title="Direct link to testpython">​</a></h3><p>Generates and runs python tests</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="environment--publishing-commands">Environment + Publishing Commands<a href="#environment--publishing-commands" class="hash-link" aria-label="Direct link to Environment + Publishing Commands" title="Direct link to Environment + Publishing Commands">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="getdatasets"><code>getDatasets</code><a href="#getdatasets" class="hash-link" aria-label="Direct link to getdatasets" title="Direct link to getdatasets">​</a></h3><p>Downloads all datasets used in tests to target folder</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="setup"><code>setup</code><a href="#setup" class="hash-link" aria-label="Direct link to setup" title="Direct link to setup">​</a></h3><p>Combination of <code>compile</code>, <code>test:compile</code>, <code>it:compile</code>, <code>getDatasets</code></p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="package"><code>package</code><a href="#package" class="hash-link" aria-label="Direct link to package" title="Direct link to package">​</a></h3><p>Packages the library into a jar</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishblob"><code>publishBlob</code><a href="#publishblob" class="hash-link" aria-label="Direct link to publishblob" title="Direct link to publishblob">​</a></h3><p>Publishes Jar to SynapseML&#x27;s Azure blob-based Maven repo. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishlocal"><code>publishLocal</code><a href="#publishlocal" class="hash-link" aria-label="Direct link to publishlocal" title="Direct link to publishlocal">​</a></h3><p>Publishes library to the local Maven repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishdocs"><code>publishDocs</code><a href="#publishdocs" class="hash-link" aria-label="Direct link to publishdocs" title="Direct link to publishdocs">​</a></h3><p>Publishes scala and python doc to SynapseML&#x27;s Azure storage account. (Requires Keys)</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="publishsigned"><code>publishSigned</code><a href="#publishsigned" class="hash-link" aria-label="Direct link to publishsigned" title="Direct link to publishsigned">​</a></h3><p>Publishes the library to Sonatype staging repo</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="sonatyperelease"><code>sonatypeRelease</code><a href="#sonatyperelease" class="hash-link" aria-label="Direct link to sonatyperelease" title="Direct link to sonatyperelease">​</a></h3><p>Promotes the published Sonatype artifact</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Reference/Contributor Guide/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Contributor Guide</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#scala-build-commands" class="table-of-contents__link toc-highlight">Scala build commands</a><ul><li><a href="#compile-testcompile-and-itcompile" class="table-of-contents__link toc-highlight"><code>compile</code>, <code>test:compile</code> and <code>it:compile</code></a></li><li><a href="#test" class="table-of-contents__link toc-highlight"><code>test</code></a></li><li><a href="#scalastyle" class="table-of-contents__link toc-highlight"><code>scalastyle</code></a></li><li><a href="#testscalastyle" class="table-of-contents__link toc-highlight"><code>test:scalastyle</code></a></li><li><a href="#unidoc" class="table-of-contents__link toc-highlight"><code>unidoc</code></a></li></ul></li><li><a href="#python-commands" class="table-of-contents__link toc-highlight">Python Commands</a><ul><li><a href="#createcondaenv" class="table-of-contents__link toc-highlight"><code>createCondaEnv</code></a></li><li><a href="#cleancondaenv" class="table-of-contents__link toc-highlight"><code>cleanCondaEnv</code></a></li><li><a href="#packagepython" class="table-of-contents__link toc-highlight"><code>packagePython</code></a></li><li><a href="#generatepythondoc" class="table-of-contents__link toc-highlight"><code>generatePythonDoc</code></a></li><li><a href="#installpippackage" class="table-of-contents__link toc-highlight"><code>installPipPackage</code></a></li><li><a href="#testpython" class="table-of-contents__link toc-highlight"><code>testPython</code></a></li></ul></li><li><a href="#environment--publishing-commands" class="table-of-contents__link toc-highlight">Environment + Publishing Commands</a><ul><li><a href="#getdatasets" class="table-of-contents__link toc-highlight"><code>getDatasets</code></a></li><li><a href="#setup" class="table-of-contents__link toc-highlight"><code>setup</code></a></li><li><a href="#package" class="table-of-contents__link toc-highlight"><code>package</code></a></li><li><a href="#publishblob" class="table-of-contents__link toc-highlight"><code>publishBlob</code></a></li><li><a href="#publishlocal" class="table-of-contents__link toc-highlight"><code>publishLocal</code></a></li><li><a href="#publishdocs" class="table-of-contents__link toc-highlight"><code>publishDocs</code></a></li><li><a href="#publishsigned" class="table-of-contents__link toc-highlight"><code>publishSigned</code></a></li><li><a href="#sonatyperelease" class="table-of-contents__link toc-highlight"><code>sonatypeRelease</code></a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/Docker Setup/index.html b/docs/next/Reference/Docker Setup/index.html
index 84e3f8efd4..9d5dc1495e 100644
--- a/docs/next/Reference/Docker Setup/index.html	
+++ b/docs/next/Reference/Docker Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -127,7 +127,7 @@
 continues to run as usual.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="further-reading">Further reading<a href="#further-reading" class="hash-link" aria-label="Direct link to Further reading" title="Direct link to Further reading">​</a></h2><p>This text briefly covers some of the useful things that you can do with the
 SynapseML Docker image (and other images in general).  You can find much more
 documentation <a href="https://docs.docker.com/" target="_blank" rel="noopener noreferrer">online</a>.</p></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Reference/Developer Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Developer Setup</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Reference/R Setup/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">R setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#quickstart-install-and-run-the-docker-image" class="table-of-contents__link toc-highlight">Quickstart: install and run the Docker image</a></li><li><a href="#running-a-specific-version" class="table-of-contents__link toc-highlight">Running a specific version</a></li><li><a href="#a-more-practical-example" class="table-of-contents__link toc-highlight">A more practical example</a></li><li><a href="#running-the-container-as-a-server" class="table-of-contents__link toc-highlight">Running the container as a server</a></li><li><a href="#running-other-commands-in-an-active-container" class="table-of-contents__link toc-highlight">Running other commands in an active container</a></li><li><a href="#running-other-spark-executables" class="table-of-contents__link toc-highlight">Running other Spark executables</a></li><li><a href="#updating-the-synapseml-image" class="table-of-contents__link toc-highlight">Updating the SynapseML image</a></li><li><a href="#a-note-about-security" class="table-of-contents__link toc-highlight">A note about security</a></li><li><a href="#further-reading" class="table-of-contents__link toc-highlight">Further reading</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/Dotnet Setup/index.html b/docs/next/Reference/Dotnet Setup/index.html
index 6cdaba728c..f9ea5fbefd 100644
--- a/docs/next/Reference/Dotnet Setup/index.html	
+++ b/docs/next/Reference/Dotnet Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -52,7 +52,7 @@
 Otherwise, you&#x27;d have to use the full path (for example, C:\bin\apache-spark\bin\spark-submit or ~/spark/bin/spark-submit).</p></div></div><p>When your app runs, the sentiment analysis result is written to the console.</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|                                     text|language|error|                                         sentiment|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|          I am so happy today, its sunny!|   en-US| null|[{positive, null, {0.99, 0.0, 0.0}, [{I am so h...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|I am frustrated by this rush hour traffic|   en-US| null|[{negative, null, {0.0, 0.0, 0.99}, [{I am frus...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| The Azure AI services on spark aint bad|   en-US| null|[{positive, null, {0.99, 0.01, 0.00}, [{The cogn...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+-----------------------------------------+--------+-----+--------------------------------------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Congratulations! You successfully authored and ran a .NET for SynapseML app.
 Refer to the <a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/dotnet/index.html" target="_blank" rel="noopener noreferrer">developer docs</a> for API guidance.</p><h2 class="anchor anchorWithStickyNavbar_LWe7" id="next">Next<a href="#next" class="hash-link" aria-label="Direct link to Next" title="Direct link to Next">​</a></h2><ul><li>Refer to this <a href="https://docs.microsoft.com/en-us/dotnet/spark/tutorials/databricks-deployment" target="_blank" rel="noopener noreferrer">tutorial</a> for deploying a .NET app to Databricks.</li><li>You could download compatible <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/install-worker.sh" target="_blank" rel="noopener noreferrer">install-worker.sh</a>
 and <a href="https://mmlspark.blob.core.windows.net/publicwasb/dotnet/db-init.sh" target="_blank" rel="noopener noreferrer">db-init.sh</a> files needed for deployment on Databricks.</li></ul></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#1-install-net" class="table-of-contents__link toc-highlight">1. Install .NET</a></li><li><a href="#2-install-java" class="table-of-contents__link toc-highlight">2. Install Java</a></li><li><a href="#3-install-apache-spark" class="table-of-contents__link toc-highlight">3. Install Apache Spark</a></li><li><a href="#4-install-net-for-apache-spark" class="table-of-contents__link toc-highlight">4. Install .NET for Apache Spark</a></li><li><a href="#5-install-winutils-windows-only" class="table-of-contents__link toc-highlight">5. Install WinUtils (Windows Only)</a></li><li><a href="#6-set-dotnet_worker_dir-and-check-dependencies" class="table-of-contents__link toc-highlight">6. Set DOTNET_WORKER_DIR and check dependencies</a></li></ul></li><li><a href="#write-a-net-for-synapseml-app" class="table-of-contents__link toc-highlight">Write a .NET for SynapseML App</a><ul><li><a href="#1-create-a-console-app" class="table-of-contents__link toc-highlight">1. Create a console app</a></li><li><a href="#2-install-nuget-package" class="table-of-contents__link toc-highlight">2. Install NuGet package</a></li><li><a href="#3-write-your-app" class="table-of-contents__link toc-highlight">3. Write your app</a></li><li><a href="#4-run-your-net-app" class="table-of-contents__link toc-highlight">4. Run your .NET App</a></li></ul></li><li><a href="#next" class="table-of-contents__link toc-highlight">Next</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/Quickstart - LightGBM in Dotnet/index.html b/docs/next/Reference/Quickstart - LightGBM in Dotnet/index.html
index eaca1d0330..424535f12b 100644
--- a/docs/next/Reference/Quickstart - LightGBM in Dotnet/index.html	
+++ b/docs/next/Reference/Quickstart - LightGBM in Dotnet/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Reference/Quickstart - LightGBM in Dotnet/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Reference/Quickstart - LightGBM in Dotnet/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Reference/Quickstart - LightGBM in Dotnet/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Reference/Quickstart - LightGBM in Dotnet/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Reference/Quickstart - LightGBM in Dotnet/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Reference/Quickstart - LightGBM in Dotnet/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><main class="docMainContainer_gTbr docMainContainerEnhanced_Uz_u"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Reference/Quickstart - LightGBM in Dotnet/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Quickstart - LightGBM in Dotnet</h1></header><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Make sure you have followed the guidance in <a href="/SynapseML/docs/next/Reference/Dotnet Setup/">.NET installation</a> before jumping into this example.</p></div></div><h2 class="anchor anchorWithStickyNavbar_LWe7" id="classification-with-lightgbmclassifier">Classification with LightGBMClassifier<a href="#classification-with-lightgbmclassifier" class="hash-link" aria-label="Direct link to Classification with LightGBMClassifier" title="Direct link to Classification with LightGBMClassifier">​</a></h2><p>Install NuGet packages by running following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package Microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Spark </span><span class="token operator">--</span><span class="token plain">version 2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Lightgbm </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">dotnet add package SynapseML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">Core </span><span class="token operator">--</span><span class="token plain">version 1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Use the following code in your main program file:</p><div class="language-csharp codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-csharp codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">System</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Collections</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Generic</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Lightgbm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Synapse</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">ML</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">using</span><span class="token plain"> </span><span class="token namespace">Microsoft</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Spark</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Sql</span><span class="token namespace punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token namespace">Types</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">namespace</span><span class="token plain"> </span><span class="token namespace">SynapseMLApp</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token keyword">class</span><span class="token plain"> </span><span class="token class-name">Program</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token keyword">static</span><span class="token plain"> </span><span class="token return-type class-name keyword">void</span><span class="token plain"> </span><span class="token function" style="color:rgb(166, 226, 46)">Main</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token class-name keyword">string</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> args</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">SparkSession</span><span class="token plain"> spark </span><span class="token operator">=</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                SparkSession</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">AppName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;LightGBMExample&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">GetOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Load Data</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token plain"> df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Read</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Option</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;inferSchema&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Parquet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;wasbs://publicwasb@mmlspark.blob.core.windows.net/AdultCensusIncome.parquet&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Limit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featureColumns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">string</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;age&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;workclass&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;fnlwgt&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;education-num&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;marital-status&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;occupation&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;relationship&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;race&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sex&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-gain&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;capital-loss&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;hours-per-week&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;native-country&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Transform features</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">featureColumns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">true</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">14</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> dfTrans </span><span class="token operator">=</span><span class="token plain"> featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">WithColumn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">When</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Functions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Col</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;income&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Contains</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;&lt;&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Otherwise</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name">DataFrame</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> dfs </span><span class="token operator">=</span><span class="token plain"> dfTrans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">RandomSplit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name keyword">double</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token constructor-invocation class-name punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token number" style="color:rgb(174, 129, 255)">0.75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">123</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> trainDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> testDf </span><span class="token operator">=</span><span class="token plain"> dfs</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Create LightGBMClassifier</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token keyword">new</span><span class="token plain"> </span><span class="token constructor-invocation class-name">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">200</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;label&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">SetFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Fit the model</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token class-name keyword">var</span><span class="token plain"> lightGBMClassificationModel </span><span class="token operator">=</span><span class="token plain"> lightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">trainDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Apply transformation and displayresults</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            lightGBMClassificationModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">testDf</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Show</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token comment" style="color:rgb(136, 132, 111)">// Stop Spark session</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token function" style="color:rgb(166, 226, 46)">Stop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>Run <code>dotnet build</code> to build the project. Then navigate to build output directory, and run following command:</p><div class="language-powershell codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-powershell codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit </span><span class="token operator">--</span><span class="token keyword">class</span><span class="token plain"> org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">deploy</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dotnet</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">DotnetRunner </span><span class="token operator">--</span><span class="token plain">packages com</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">microsoft</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">azure:synapseml_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12:1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain">org</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">apache</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">hadoop:hadoop-azure:3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1 </span><span class="token operator">--</span><span class="token plain">master local microsoft-spark-3-2_2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">12-2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">jar dotnet SynapseMLApp</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dll</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><div class="theme-admonition theme-admonition-note alert alert--secondary admonition_LlT9"><div class="admonitionHeading_tbUL"><span class="admonitionIcon_kALy"><svg viewBox="0 0 14 16"><path fill-rule="evenodd" d="M6.3 5.69a.942.942 0 0 1-.28-.7c0-.28.09-.52.28-.7.19-.18.42-.28.7-.28.28 0 .52.09.7.28.18.19.28.42.28.7 0 .28-.09.52-.28.7a1 1 0 0 1-.7.3c-.28 0-.52-.11-.7-.3zM8 7.99c-.02-.25-.11-.48-.31-.69-.2-.19-.42-.3-.69-.31H6c-.27.02-.48.13-.69.31-.2.2-.3.44-.31.69h1v3c.02.27.11.5.31.69.2.2.42.31.69.31h1c.27 0 .48-.11.69-.31.2-.19.3-.42.31-.69H8V7.98v.01zM7 2.3c-3.14 0-5.7 2.54-5.7 5.68 0 3.14 2.56 5.7 5.7 5.7s5.7-2.55 5.7-5.7c0-3.15-2.56-5.69-5.7-5.69v.01zM7 .98c3.86 0 7 3.14 7 7s-3.14 7-7 7-7-3.12-7-7 3.14-7 7-7z"></path></svg></span>note</div><div class="admonitionContent_S0QG"><p>Here we added two packages: synapseml_2.12 for SynapseML&#x27;s scala source, and hadoop-azure to support reading files from ADLS.</p></div></div><p>Expected output:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">|age|workclass|fnlwgt|    education|education-num|marital-status|        occupation|   relationship|               race|    sex|capital-gain|capital-loss|hours-per-week|native-country|income|            features|label|       rawPrediction|         probability|prediction|      leafPrediction|        featuresShap|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|        ?|634226|         10th|            6| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          17.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[9.37122343731523...|[0.99991486808581...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0560742274706...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private| 73145|          9th|            5| Never-married|      Craft-repair|      Own-child|              White| Female|           0|           0|          16.0| United-States| &lt;=50K|(61,[7,9,11,15,17...|  0.0|[12.7512760001880...|[0.99999710138899...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1657810433238...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|150106|         10th|            6| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[12.7676985938038...|[0.99999714860282...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1276877355292...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|151141|         11th|            7| Never-married| Handlers-cleaners|      Own-child|              White|   Male|           0|           0|          15.0| United-States| &lt;=50K|(61,[8,9,11,15,17...|  0.0|[12.1656242513070...|[0.99999479363924...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1279828578119...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 17|  Private|327127|         11th|            7| Never-married|  Transport-moving|      Own-child|              White|   Male|           0|           0|          20.0| United-States| &lt;=50K|(61,[1,9,11,15,17...|  0.0|[12.9962776686392...|[0.99999773124636...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1164691543415...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|        ?|171088| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[12.9400428266629...|[0.99999760000817...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1554829578661...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|115839|         12th|            8| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[11.8393032168619...|[0.99999278472630...|       0.0|[0.0,0.0,0.0,0.0,...|[0.44080835709189...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|133055|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.5747235180479...|[0.99999059936124...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1415862541824...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|169745|      7th-8th|            4| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[11.8316427733613...|[0.99999272924226...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1527378526573...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|177648|      HS-grad|            9| Never-married|             Sales|      Own-child|              White| Female|           0|           0|          25.0| United-States| &lt;=50K|(61,[5,9,11,15,17...|  0.0|[10.0820248199174...|[0.99995817710510...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1151843103241...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|188241|         11th|            7| Never-married|     Other-service|      Own-child|              White|   Male|           0|           0|          16.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4049945509280...|[0.99996972005153...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1356854966291...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|200603|      HS-grad|            9| Never-married|      Adm-clerical| Other-relative|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[12.1354343020828...|[0.99999463406365...|       0.0|[0.0,0.0,0.0,0.0,...|[0.53241098695335...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|210026|         10th|            6| Never-married|     Other-service| Other-relative|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.3692360082180...|[0.99999575275599...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1275208795564...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 18|  Private|447882| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2514945786032...|[0.99996469655062...|       0.0|[0.0,0.0,0.0,0.0,...|[0.36497782752201...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|        ?|242001| Some-college|           10| Never-married|                 ?|      Own-child|              White| Female|           0|           0|          40.0| United-States| &lt;=50K|(61,[7,9,11,15,20...|  0.0|[13.9439986622060...|[0.99999912057674...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1265631737386...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 63814| Some-college|           10| Never-married|      Adm-clerical|  Not-in-family|              White| Female|           0|           0|          18.0| United-States| &lt;=50K|(61,[0,9,11,15,17...|  0.0|[10.2057742895673...|[0.99996304506073...|       0.0|[0.0,0.0,0.0,0.0,...|[0.77645146059597...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 83930|      HS-grad|            9| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          20.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[10.4771335467356...|[0.99997182742919...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1625827100973...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private| 86150|         11th|            7| Never-married|             Sales|      Own-child| Asian-Pac-Islander| Female|           0|           0|          19.0|   Philippines| &lt;=50K|(61,[5,9,14,15,17...|  0.0|[12.0241839747799...|[0.99999400263272...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1532111483051...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|189574|      HS-grad|            9| Never-married|     Other-service|  Not-in-family|              White| Female|           0|           0|          30.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[9.53742673004733...|[0.99992790305091...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.0988907054317...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">| 19|  Private|219742| Some-college|           10| Never-married|     Other-service|      Own-child|              White| Female|           0|           0|          15.0| United-States| &lt;=50K|(61,[3,9,11,15,17...|  0.0|[12.8625329757574...|[0.99999740658642...|       0.0|[0.0,0.0,0.0,0.0,...|[-0.1922327651359...|</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">+---+---------+------+-------------+-------------+--------------+------------------+---------------+-------------------+-------+------------+------------+--------------+--------------+------+--------------------+-----+--------------------+--------------------+----------+--------------------+--------------------+</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#classification-with-lightgbmclassifier" class="table-of-contents__link toc-highlight">Classification with LightGBMClassifier</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Reference/R Setup/index.html b/docs/next/Reference/R Setup/index.html
index 0212d0eade..45e5275c53 100644
--- a/docs/next/Reference/R Setup/index.html	
+++ b/docs/next/Reference/R Setup/index.html	
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -32,7 +32,7 @@
 process</a>.  To get a quick build, start at the root
 of the synapseml directory, and find the generated files. For instance,
 to find the R files for deep-learning, run</p><div class="language-bash codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-bash codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">sbt packageR</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token function" style="color:rgb(166, 226, 46)">ls</span><span class="token plain"> ./deep-learning/target/scala-2.12/generated/src/R/synapseml/R</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can then run R in a terminal and install the above files directly:</p><div class="language-R codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-R codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">devtools::install_local(&quot;./deep-learning/target/scala-2.12/generated/src/R/synapseml/R&quot;)</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">...</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Reference/Docker Setup/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Docker Setup</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#importing-libraries-and-setting-up-spark-context" class="table-of-contents__link toc-highlight">Importing libraries and setting up spark context</a></li></ul></li><li><a href="#example" class="table-of-contents__link toc-highlight">Example</a></li><li><a href="#azure-databricks" class="table-of-contents__link toc-highlight">Azure Databricks</a></li><li><a href="#building-from-source" class="table-of-contents__link toc-highlight">Building from Source</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Use with MLFlow/Autologging/index.html b/docs/next/Use with MLFlow/Autologging/index.html
index 446c204fc5..0ad59d73ba 100644
--- a/docs/next/Use with MLFlow/Autologging/index.html	
+++ b/docs/next/Use with MLFlow/Autologging/index.html	
@@ -16,15 +16,15 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Autologging/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Autologging/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Autologging/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Autologging/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Autologging/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Autologging/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Use with MLFlow/Autologging/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Autologging/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Autologging</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Autologging</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="automatic-logging">Automatic Logging<a href="#automatic-logging" class="hash-link" aria-label="Direct link to Automatic Logging" title="Direct link to Automatic Logging">​</a></h2><p><a href="https://www.mlflow.org/docs/latest/tracking.html#automatic-logging" target="_blank" rel="noopener noreferrer">MLflow automatic logging</a> allows you to log metrics, parameters, and models without the need for explicit log statements.
 SynapseML supports autologging for every model in the library.</p><p>To enable autologging for SynapseML:</p><ol><li>Download this customized <a href="https://mmlspark.blob.core.windows.net/publicwasb/log_model_allowlist.txt" target="_blank" rel="noopener noreferrer">log_model_allowlist file</a> and put it at a place that your code has access to.
 For example:</li></ol><ul><li>In Synapse <code>wasb://&lt;containername&gt;@&lt;accountname&gt;.blob.core.windows.net/PATH_TO_YOUR/log_model_allowlist.txt</code></li><li>In Databricks <code>/dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</code>.</li></ul><ol start="2"><li>Set spark configuration <code>spark.mlflow.pysparkml.autolog.logModelAllowlistFile</code> to the path of your <code>log_model_allowlist.txt</code> file.</li><li>Call <code>mlflow.pyspark.ml.autolog()</code> before your training code to enable autologging for all supported models.</li></ol><p>Note:</p><ol><li>If you want to support autologging of PySpark models not present in the log_model_allowlist file, you can add such models to the file.</li><li>If you&#x27;ve enabled autologging, then don&#x27;t write explicit <code>with mlflow.start_run()</code> as it might cause multiple runs for one single model or one run for multiple models.</li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="configuration-process-in-databricks-as-an-example">Configuration process in Databricks as an example<a href="#configuration-process-in-databricks-as-an-example" class="hash-link" aria-label="Direct link to Configuration process in Databricks as an example" title="Direct link to Configuration process in Databricks as an example">​</a></h2><ol><li>Install latest MLflow via <code>%pip install mlflow</code></li><li>Upload your customized <code>log_model_allowlist.txt</code> file to dbfs by clicking File/Upload Data button on Databricks UI.</li><li>Set Cluster Spark configuration following <a href="https://docs.microsoft.com/en-us/azure/databricks/clusters/configure#spark-configuration" target="_blank" rel="noopener noreferrer">this documentation</a></li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">spark.mlflow.pysparkml.autolog.logModelAllowlistFile /dbfs/FileStore/PATH_TO_YOUR/log_model_allowlist.txt</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><ol start="4"><li>Run the following line before your training code executes.</li></ol><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow.pyspark.ml.autolog()</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>You can customize how autologging works by supplying appropriate <a href="https://www.mlflow.org/docs/latest/python_api/mlflow.pyspark.ml.html#mlflow.pyspark.ml.autolog" target="_blank" rel="noopener noreferrer">parameters</a>.</p><ol start="5"><li>To find your experiment&#x27;s results via the <code>Experiments</code> tab of the MLFlow UI.<img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/adb_experiments.png" width="1200" class="img_ev3q"></li></ol><h2 class="anchor anchorWithStickyNavbar_LWe7" id="example-for-conditionalknnmodel">Example for ConditionalKNNModel<a href="#example-for-conditionalknnmodel" class="hash-link" aria-label="Direct link to Example for ConditionalKNNModel" title="Direct link to Example for ConditionalKNNModel">​</a></h2><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">linalg </span><span class="token keyword">import</span><span class="token plain"> Vectors</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">nn </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">12.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">14.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">18.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnn </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ConditionalKNN</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">cnnm </span><span class="token operator">=</span><span class="token plain"> cnn</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">test_df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">4.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">6.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">8.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Vectors</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">dense</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">2.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">10.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;foo&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;values&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conditioner&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">cnnm</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><p>This code should log one run with a ConditionalKNNModel artifact and its parameters.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/autologgingRunSample.png" width="1200" class="img_ev3q"></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Install</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Deploy Models/Overview/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">About</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#automatic-logging" class="table-of-contents__link toc-highlight">Automatic Logging</a></li><li><a href="#configuration-process-in-databricks-as-an-example" class="table-of-contents__link toc-highlight">Configuration process in Databricks as an example</a></li><li><a href="#example-for-conditionalknnmodel" class="table-of-contents__link toc-highlight">Example for ConditionalKNNModel</a></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Use with MLFlow/Install/index.html b/docs/next/Use with MLFlow/Install/index.html
index 38c73595d5..d6b063c936 100644
--- a/docs/next/Use with MLFlow/Install/index.html	
+++ b/docs/next/Use with MLFlow/Install/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Use with MLFlow/Install/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Install/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Install/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Install/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Install/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Install/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Install/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Use with MLFlow/Install/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Install/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Install</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="theme-doc-markdown markdown"><header><h1>Install</h1></header></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Use with MLFlow/Overview/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Overview</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Use with MLFlow/Autologging/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Autologging</div></a></nav></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/docs/next/Use with MLFlow/Overview/index.html b/docs/next/Use with MLFlow/Overview/index.html
index 5d7853d409..5becee7b03 100644
--- a/docs/next/Use with MLFlow/Overview/index.html	
+++ b/docs/next/Use with MLFlow/Overview/index.html	
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/next/Overview/">Next</a><ul class="dropdown__menu"><li><a aria-current="page" class="dropdown__link dropdown__link--active" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Use with MLFlow/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Use with MLFlow/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Use with MLFlow/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Use with MLFlow/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Use with MLFlow/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Use with MLFlow/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/docs/next/Use with MLFlow/Overview/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0 docsWrapper_BCFX"><button aria-label="Scroll back to top" class="clean-btn theme-back-to-top-button backToTopButton_sjWU" type="button"></button><div class="docPage__5DB"><aside class="theme-doc-sidebar-container docSidebarContainer_b6E3"><div class="sidebarViewport_Xe31"><div class="sidebar_njMd"><nav aria-label="Docs sidebar" class="menu thin-scrollbar menu_SIkG menuWithAnnouncementBar_GW3s"><ul class="theme-doc-sidebar-menu menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-1 menu__list-item"><a class="menu__link" href="/SynapseML/docs/next/Overview/">What is SynapseML?</a></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Get Started/Create a Spark Cluster/">Get Started</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Explore Algorithms/LightGBM/Overview/">Explore Algorithms</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret menu__link--active" aria-expanded="true" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Use with MLFlow</a></div><ul style="display:block;overflow:visible;height:auto" class="menu__list"><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link menu__link--active" aria-current="page" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Overview/">Overview</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Install/">Install</a></li><li class="theme-doc-sidebar-item-link theme-doc-sidebar-item-link-level-2 menu__list-item"><a class="menu__link" tabindex="0" href="/SynapseML/docs/next/Use with MLFlow/Autologging/">Autologging</a></li></ul></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Deploy Models/Overview/">Deploy Models</a></div></li><li class="theme-doc-sidebar-item-category theme-doc-sidebar-item-category-level-1 menu__list-item menu__list-item--collapsed"><div class="menu__list-item-collapsible"><a class="menu__link menu__link--sublist menu__link--sublist-caret" aria-expanded="false" href="/SynapseML/docs/next/Reference/Contributor Guide/">Reference</a></div></li></ul></nav></div></div></aside><main class="docMainContainer_gTbr"><div class="container padding-top--md padding-bottom--lg"><div class="row"><div class="col docItemCol_VOVn"><div class="theme-doc-version-banner alert alert--warning margin-bottom--md" role="alert"><div>This is unreleased documentation for <!-- -->SynapseML<!-- --> <b>Next</b> version.</div><div class="margin-top--md">For up-to-date documentation, see the <b><a href="/SynapseML/docs/Use with MLFlow/Overview/">latest version</a></b> (<!-- -->1.0.4<!-- -->).</div></div><div class="docItemContainer_Djhp"><article><nav class="theme-doc-breadcrumbs breadcrumbsContainer_Z_bl" aria-label="Breadcrumbs"><ul class="breadcrumbs" itemscope="" itemtype="https://schema.org/BreadcrumbList"><li class="breadcrumbs__item"><a aria-label="Home page" class="breadcrumbs__link" href="/SynapseML/"><svg viewBox="0 0 24 24" class="breadcrumbHomeIcon_YNFT"><path d="M10 19v-5h4v5c0 .55.45 1 1 1h3c.55 0 1-.45 1-1v-7h1.7c.46 0 .68-.57.33-.87L12.67 3.6c-.38-.34-.96-.34-1.34 0l-8.36 7.53c-.34.3-.13.87.33.87H5v7c0 .55.45 1 1 1h3c.55 0 1-.45 1-1z" fill="currentColor"></path></svg></a></li><li class="breadcrumbs__item"><span class="breadcrumbs__link">Use with MLFlow</span><meta itemprop="position" content="1"></li><li itemscope="" itemprop="itemListElement" itemtype="https://schema.org/ListItem" class="breadcrumbs__item breadcrumbs__item--active"><span class="breadcrumbs__link" itemprop="name">Overview</span><meta itemprop="position" content="2"></li></ul></nav><span class="theme-doc-version-badge badge badge--secondary">Version: Next</span><div class="tocCollapsible_ETCw theme-doc-toc-mobile tocMobile_ITEo"><button type="button" class="clean-btn tocCollapsibleButton_TO0P">On this page</button></div><div class="theme-doc-markdown markdown"><header><h1>Overview</h1></header><h2 class="anchor anchorWithStickyNavbar_LWe7" id="what-is-mlflow">What is MLflow<a href="#what-is-mlflow" class="hash-link" aria-label="Direct link to What is MLflow" title="Direct link to What is MLflow">​</a></h2><p><a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">MLflow</a> is a platform to streamline machine learning development, including tracking experiments, packaging code into reproducible runs, and sharing and deploying models. MLflow offers a set of lightweight APIs that can be used with any existing machine learning application or library, for instance TensorFlow, PyTorch, XGBoost, etc. It runs wherever you currently run ML code, for example, in notebooks, standalone applications or the cloud. MLflow&#x27;s current components are:</p><ul><li><a href="https://mlflow.org/docs/latest/tracking.html" target="_blank" rel="noopener noreferrer">MLflow Tracking</a>: An API to log parameters, code, and results in machine learning experiments and compare them using an interactive UI.</li><li><a href="https://mlflow.org/docs/latest/projects.html" target="_blank" rel="noopener noreferrer">MLflow Projects</a>: A code packaging format for reproducible runs using Conda and Docker, so you can share your ML code with others.</li><li><a href="https://mlflow.org/docs/latest/models.html" target="_blank" rel="noopener noreferrer">MLflow Models</a>: A model packaging format and tools that let you easily deploy the same model from any ML library for both batch and real-time scoring. It supports platforms such as Docker, Apache Spark, Azure ML and AWS SageMaker.</li><li><a href="https://mlflow.org/docs/latest/model-registry.html" target="_blank" rel="noopener noreferrer">MLflow Model Registry</a>: A centralized model store, set of APIs, and UI, to collaboratively manage the full lifecycle of MLflow Models.</li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="installation">Installation<a href="#installation" class="hash-link" aria-label="Direct link to Installation" title="Direct link to Installation">​</a></h2><p>Install MLflow from PyPI via <code>pip install mlflow</code></p><p>MLflow requires <code>conda</code> to be on the <code>PATH</code> for the projects feature.</p><p>Learn more about MLflow on their <a href="https://github.com/mlflow/mlflow" target="_blank" rel="noopener noreferrer">GitHub page</a>.</p><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-databricks">Install Mlflow on Databricks<a href="#install-mlflow-on-databricks" class="hash-link" aria-label="Direct link to Install Mlflow on Databricks" title="Direct link to Install Mlflow on Databricks">​</a></h3><p>If you&#x27;re using Databricks, install Mlflow with this command:</p><div class="codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-text codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain"># run this so that Mlflow is installed on workers besides driver</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">%pip install mlflow</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="install-mlflow-on-synapse">Install Mlflow on Synapse<a href="#install-mlflow-on-synapse" class="hash-link" aria-label="Direct link to Install Mlflow on Synapse" title="Direct link to Install Mlflow on Synapse">​</a></h3><p>To log model with Mlflow, you need to create an Azure Machine Learning workspace and link it with your Synapse workspace.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-azure-machine-learning-workspace">Create Azure Machine Learning Workspace<a href="#create-azure-machine-learning-workspace" class="hash-link" aria-label="Direct link to Create Azure Machine Learning Workspace" title="Direct link to Create Azure Machine Learning Workspace">​</a></h4><p>Follow this document to create <a href="https://learn.microsoft.com/en-us/azure/machine-learning/quickstart-create-resources#create-the-workspace" target="_blank" rel="noopener noreferrer">AML workspace</a>. You don&#x27;t need to create compute instance and compute clusters.</p><h4 class="anchor anchorWithStickyNavbar_LWe7" id="create-an-azure-ml-linked-service">Create an Azure ML Linked Service<a href="#create-an-azure-ml-linked-service" class="hash-link" aria-label="Direct link to Create an Azure ML Linked Service" title="Direct link to Create an Azure ML Linked Service">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_1.png" width="600" class="img_ev3q"><ul><li>In the Synapse workspace, go to <strong>Manage</strong> -&gt; <strong>External connections</strong> -&gt; <strong>Linked services</strong>, select <strong>+ New</strong></li><li>Select the workspace you want to log the model in and create the linked service. You need the <strong>name of the linked service</strong> to set up connection.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="auth-synapse-workspace">Auth Synapse Workspace<a href="#auth-synapse-workspace" class="hash-link" aria-label="Direct link to Auth Synapse Workspace" title="Direct link to Auth Synapse Workspace">​</a></h4><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/ml_linked_service_2.png" width="600" class="img_ev3q"><ul><li>Go to the <strong>Azure Machine Learning workspace</strong> resource -&gt; <strong>access control (IAM)</strong> -&gt; <strong>Role assignment</strong>, select <strong>+ Add</strong>, choose <strong>Add role assignment</strong></li><li>Choose <strong>contributor</strong>, select next</li><li>In members page, choose <strong>Managed identity</strong>, select  <strong>+ select members</strong>. Under <strong>managed identity</strong>, choose Synapse workspace. Under <strong>Select</strong>, choose the workspace you run your experiment on. Click <strong>Select</strong>, <strong>Review + assign</strong>.</li></ul><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-with-linked-service">Use MLFlow in Synapse with Linked Service<a href="#use-mlflow-in-synapse-with-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse with Linked Service" title="Direct link to Use MLFlow in Synapse with Linked Service">​</a></h4><p>Set up connection</p><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#AML workspace authentication using linked service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> notebookutils</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">mssparkutils </span><span class="token keyword">import</span><span class="token plain"> azureML</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">linked_service_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YourLinkedServiceName&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">ws </span><span class="token operator">=</span><span class="token plain"> azureML</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getWorkspace</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">linked_service_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ws</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">get_mlflow_tracking_uri</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token comment" style="color:rgb(136, 132, 111)">#Set MLflow experiment. </span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">experiment_name </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapse-mlflow-experiment&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">set_experiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">experiment_name</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h4 class="anchor anchorWithStickyNavbar_LWe7" id="use-mlflow-in-synapse-without-a-linked-service">Use MLFlow in Synapse without a Linked Service<a href="#use-mlflow-in-synapse-without-a-linked-service" class="hash-link" aria-label="Direct link to Use MLFlow in Synapse without a Linked Service" title="Direct link to Use MLFlow in Synapse without a Linked Service">​</a></h4><p>Once you create an AML workspace, you can obtain the MLflow tracking URL directly. The AML start page is where you can locate the MLflow tracking URL.</p><img loading="lazy" src="https://mmlspark.blob.core.windows.net/graphics/Documentation/mlflow_tracking_url.png" width="600" class="img_ev3q">You can set it tracking url with ```python mlflow.set_tracking_uri(&quot;your mlflow tracking url&quot;) ```<h2 class="anchor anchorWithStickyNavbar_LWe7" id="mlflow-api-reference">MLFlow API Reference<a href="#mlflow-api-reference" class="hash-link" aria-label="Direct link to MLFlow API Reference" title="Direct link to MLFlow API Reference">​</a></h2><ul><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.save_model" target="_blank" rel="noopener noreferrer">mlflow.spark.save_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.log_model" target="_blank" rel="noopener noreferrer">mlflow.spark.log_model</a></li><li><a href="https://www.mlflow.org/docs/latest/python_api/mlflow.spark.html#mlflow.spark.load_model" target="_blank" rel="noopener noreferrer">mlflow.spark.load_model</a></li><li><a href="https://mlflow.org/docs/latest/python_api/mlflow.html#mlflow.log_metric" target="_blank" rel="noopener noreferrer">mlflow.log_metric</a></li></ul><h2 class="anchor anchorWithStickyNavbar_LWe7" id="examples">Examples<a href="#examples" class="hash-link" aria-label="Direct link to Examples" title="Direct link to Examples">​</a></h2><h3 class="anchor anchorWithStickyNavbar_LWe7" id="lightgbmclassifier">LightGBMClassifier<a href="#lightgbmclassifier" class="hash-link" aria-label="Direct link to LightGBMClassifier" title="Direct link to LightGBMClassifier">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">featurize </span><span class="token keyword">import</span><span class="token plain"> Featurize</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">train </span><span class="token keyword">import</span><span class="token plain"> ComputeModelStatistics</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    feature_columns </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Number of times pregnant&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Plasma glucose concentration a 2 hours in an oral glucose tolerance test&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diastolic blood pressure (mm Hg)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Triceps skin fold thickness (mm)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;2-Hour serum insulin (mu U/ml)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Body mass index (weight in kg/(height in m)^2)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Diabetes pedigree function&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;Age (years)&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">131</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">66</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">40</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.196</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">194</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">68</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.745</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">139</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">134</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">130</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.542</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">124</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.282</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">34</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">43.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.021</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">110</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">20</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.118</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">127</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">275</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">25</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">104</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">37</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">33.6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.51</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">120</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">54</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.455</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">178</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">39.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.331</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">41</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">88</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">58</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">16</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.4</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.766</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">22</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">91</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">24</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.192</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">21</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">101</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">76</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">48</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">180</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">32.9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.171</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">63</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">73</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">26.8</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.268</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">27</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">158</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">70</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">328</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35.5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.344</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">105</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">75</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.56</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">53</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">84</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">72</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.297</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">46</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">9</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">119</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">80</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">35</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29.0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.263</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">29</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">6</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">93</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">50</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">64</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">28.7</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.356</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">23</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">126</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">60</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">30.1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">0.349</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">47</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> feature_columns</span><span class="token operator">+</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">repartition</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">2</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    featurize </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">Featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">feature_columns</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOneHotEncodeCategoricals</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumFeatures</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">4096</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df_trans </span><span class="token operator">=</span><span class="token plain"> featurize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_classifier </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMClassifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;features&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setRawPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;rawPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setDefaultListenPort</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">12402</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">10</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setObjective</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;binary&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLabelCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;labels&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLeafPredictionCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;leafPrediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">            </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setFeaturesShapCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;featuresShap&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    lightgbm_model </span><span class="token operator">=</span><span class="token plain"> lightgbm_classifier</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.save_model to save the model to your path</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.log_model to log the model if you have a connected mlflow service</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">lightgbm_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.pyfunc.load_model to load model back as PyFuncModel and apply predict</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># Use mlflow.spark.load_model to load model back as PipelineModel and apply transform</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    predictions </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;lightgbm_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df_trans</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    metrics </span><span class="token operator">=</span><span class="token plain"> ComputeModelStatistics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">evaluationMetric</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;classification&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> labelCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;labels&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> scoredLabelsCol</span><span class="token operator">=</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;prediction&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">predictions</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">collect</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_metric</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;accuracy&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> metrics</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;accuracy&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div><h3 class="anchor anchorWithStickyNavbar_LWe7" id="azure-ai-services">Azure AI Services<a href="#azure-ai-services" class="hash-link" aria-label="Direct link to Azure AI Services" title="Direct link to Azure AI Services">​</a></h3><div class="language-python codeBlockContainer_Ckt0 theme-code-block" style="--prism-color:#f8f8f2;--prism-background-color:#272822"><div class="codeBlockContent_biex"><pre tabindex="0" class="prism-code language-python codeBlock_bY9V thin-scrollbar"><code class="codeBlockLines_e6Vv"><span class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> mlflow</span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">services </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">with</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">start_run</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">:</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    text_key </span><span class="token operator">=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;YOUR_COG_SERVICE_SUBSCRIPTION_KEY&quot;</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    df </span><span class="token operator">=</span><span class="token plain"> spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">createDataFrame</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am so happy today, its sunny!&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;I am frustrated by this rush hour traffic&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;The cognitive services on spark aint bad&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;en-US&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    sentiment_model </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TextSentiment</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setSubscriptionKey</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">text_key</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLocation</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;eastus&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTextCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;text&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setErrorCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;error&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">                </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLanguageCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;language&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">save_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">sentiment_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    output_df </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">spark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    display</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">output_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># In order to call the predict function successfully you need to specify the</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token comment" style="color:rgb(136, 132, 111)"># outputCol name as `prediction`</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">pyfunc</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">load_model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;sentiment_model&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">predict</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">toPandas</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    prediction </span><span class="token operator">=</span><span class="token plain"> </span><span class="token builtin" style="color:rgb(174, 129, 255)">list</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">map</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token builtin" style="color:rgb(174, 129, 255)">str</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span><br></span><span class="token-line" style="color:#f8f8f2"><span class="token plain">    mlflow</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">log_param</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;prediction&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;,&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">join</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">prediction</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><br></span></code></pre><div class="buttonGroup__atx"><button type="button" aria-label="Copy code to clipboard" title="Copy" class="clean-btn"><span class="copyButtonIcons_eSgA" aria-hidden="true"><svg viewBox="0 0 24 24" class="copyButtonIcon_y97N"><path fill="currentColor" d="M19,21H8V7H19M19,5H8A2,2 0 0,0 6,7V21A2,2 0 0,0 8,23H19A2,2 0 0,0 21,21V7A2,2 0 0,0 19,5M16,1H4A2,2 0 0,0 2,3V17H4V3H16V1Z"></path></svg><svg viewBox="0 0 24 24" class="copyButtonSuccessIcon_LjdS"><path fill="currentColor" d="M21,7L9,19L3.5,13.5L4.91,12.09L9,16.17L19.59,5.59L21,7Z"></path></svg></span></button></div></div></div></div></article><nav class="pagination-nav docusaurus-mt-lg" aria-label="Docs pages"><a class="pagination-nav__link pagination-nav__link--prev" href="/SynapseML/docs/next/Explore Algorithms/Other Algorithms/Quickstart - Exploring Art Across Cultures/"><div class="pagination-nav__sublabel">Previous</div><div class="pagination-nav__label">Quickstart - Exploring Art Across Cultures</div></a><a class="pagination-nav__link pagination-nav__link--next" href="/SynapseML/docs/next/Use with MLFlow/Install/"><div class="pagination-nav__sublabel">Next</div><div class="pagination-nav__label">Install</div></a></nav></div></div><div class="col col--3"><div class="tableOfContents_bqdL thin-scrollbar theme-doc-toc-desktop"><ul class="table-of-contents table-of-contents__left-border"><li><a href="#what-is-mlflow" class="table-of-contents__link toc-highlight">What is MLflow</a></li><li><a href="#installation" class="table-of-contents__link toc-highlight">Installation</a><ul><li><a href="#install-mlflow-on-databricks" class="table-of-contents__link toc-highlight">Install Mlflow on Databricks</a></li><li><a href="#install-mlflow-on-synapse" class="table-of-contents__link toc-highlight">Install Mlflow on Synapse</a></li></ul></li><li><a href="#mlflow-api-reference" class="table-of-contents__link toc-highlight">MLFlow API Reference</a></li><li><a href="#examples" class="table-of-contents__link toc-highlight">Examples</a><ul><li><a href="#lightgbmclassifier" class="table-of-contents__link toc-highlight">LightGBMClassifier</a></li><li><a href="#azure-ai-services" class="table-of-contents__link toc-highlight">Azure AI Services</a></li></ul></li></ul></div></div></div></div></main></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/index.html b/index.html
index 1da37b64c0..18369778b8 100644
--- a/index.html
+++ b/index.html
@@ -16,8 +16,8 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
@@ -27,7 +27,7 @@
 </span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">interpretation_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">TabularSHAP</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCols</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">features</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;shapValues&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;probability&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setTargetClasses</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumSamples</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">5000</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setModel</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">model</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">input_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span></div></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_C2v_ clean-btn">Copy</button><a class="furtherButton_rET3 button button--outline button--primary" href="/SynapseML/docs/Explore Algorithms/Responsible AI/Interpreting Model Predictions/">Read more</a></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="codeBlockContainer_v98K"><div class="codeBlockContent_xAFg python"><pre class="prism-code language-python" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">lightgbm </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">quantile_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">LightGBMRegressor</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setApplication</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&#x27;quantile&#x27;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setAlpha</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setLearningRate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">0.3</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumIterations</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">100</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setNumLeaves</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">31</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">fit</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">train_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">test_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span></div></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_C2v_ clean-btn">Copy</button><a class="furtherButton_rET3 button button--outline button--primary" href="/SynapseML/docs/Explore Algorithms/LightGBM/Overview/">Read more</a></div></div></div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><div class="codeBlockContainer_v98K"><div class="codeBlockContent_xAFg python"><pre class="prism-code language-python" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token keyword">from</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">opencv </span><span class="token keyword">import</span><span class="token plain"> </span><span class="token operator">*</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain" style="display:inline-block">
 </span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">image_df </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">ImageTransformer</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setInputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">setOutputCol</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;transformed_images&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">resize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token boolean">True</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">centerCrop</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">normalize</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">mean</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.485</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.456</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.406</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> std</span><span class="token operator">=</span><span class="token punctuation" style="color:rgb(249, 38, 114)">[</span><span class="token number" style="color:rgb(174, 129, 255)">0.229</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.224</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">0.225</span><span class="token punctuation" style="color:rgb(249, 38, 114)">]</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> color_scale_factor </span><span class="token operator">=</span><span class="token plain"> </span><span class="token number" style="color:rgb(174, 129, 255)">1</span><span class="token operator">/</span><span class="token number" style="color:rgb(174, 129, 255)">255</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">    </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">transform</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">input_df</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span></div></pre><button type="button" aria-label="Copy code to clipboard" class="copyButton_C2v_ clean-btn">Copy</button><a class="furtherButton_rET3 button button--outline button--primary" href="/SynapseML/docs/Explore Algorithms/OpenCV/Image Transformations/">Read more</a></div></div></div></div></div></section></div></div></div><section class="features_cAfv"><div class="container margin-vert--md"><div class="row"><div class="col col--6"><div class="text--center"><img class="padding-vert--md featureImage_wMIZ" src="/SynapseML/img/simple.svg" alt="Simple"></div><h3>Simple</h3><p></p><p>Quickly create, train, and use distributed machine learning tools in only a few lines of code.</p><p></p></div><div class="col col--6"><div class="text--center"><img class="padding-vert--md featureImage_wMIZ" src="/SynapseML/img/scalable.svg" alt="Scalable"></div><h3>Scalable</h3><p></p><p>Scale ML workloads to hundreds of machines on your<!-- --> <a href="https://spark.apache.org/">Apache Spark</a> cluster.</p><p></p></div><div class="col col--6"><div class="text--center"><img class="padding-vert--md featureImage_wMIZ" src="/SynapseML/img/multilingual.svg" alt="Multilingual"></div><h3>Multilingual</h3><p></p><p>Use SynapseML from any Spark compatible language including Python, Scala, R, Java, .NET and C#.</p><p></p></div><div class="col col--6"><div class="text--center"><img class="padding-vert--md featureImage_wMIZ" src="/SynapseML/img/open_source.svg" alt="Open"></div><h3>Open</h3><p></p><p>SynapseML is Open Source and can be installed and used on any Spark 3 infrastructure including your local machine, Databricks, Synapse Analytics, and others.</p><p></p></div></div></div></section><div class="container"><div class="row"><div class="pitch_XV8K col"><h2>Installation</h2><p>Written in Scala, and support multiple languages.<!-- --> <a href="https://github.com/microsoft/SynapseML">Open source</a> <!-- -->and cloud native.</p><div class="tabs-container tabList__CuJ"><ul role="tablist" aria-orientation="horizontal" class="tabs"><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Synapse</li><li role="tab" tabindex="0" aria-selected="true" class="tabs__item tabItem_LNqP tabs__item--active">Fabric</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Spark Packages</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Databricks</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Docker</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">Python</li><li role="tab" tabindex="-1" aria-selected="false" class="tabs__item tabItem_LNqP">SBT</li></ul><div class="margin-top--md"><div role="tabpanel" class="tabItem_Ymn6" hidden=""><p>SynapseML can be installed on Synapse adding the following to the first cell of a notebook:</p>For Spark3.4 pools:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">%%configure -f</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conf&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.packages&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;com.microsoft.azure:synapseml_2.12:1.0.4&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.repositories&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.azureedge.net/maven&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.excludes&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.yarn.user.classpath.first&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.sql.parquet.enableVectorizedReader&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;false&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span></div></pre>For Spark3.3 pools:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">%%configure -f</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conf&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.packages&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;com.microsoft.azure:synapseml_2.12:0.11.4-spark3.3&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.repositories&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.azureedge.net/maven&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.excludes&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.yarn.user.classpath.first&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.sql.parquet.enableVectorizedReader&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;false&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span></div></pre></div><div role="tabpanel" class="tabItem_Ymn6"><p>SynapseML is preinstalled on Fabric. To install a different version, add the following to the first cell of a notebook:</p><pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">%%configure -f</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;name&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;conf&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">{</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.packages&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;com.microsoft.azure:synapseml_2.12:[THE_SYNAPSEML_VERSION_YOU_WANT]&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.repositories&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.azureedge.net/maven&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.excludes&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;org.scala-lang:scala-reflect,org.apache.spark:spark-tags_2.12,org.scalactic:scalactic_2.12,org.scalatest:scalatest_2.12,com.fasterxml.jackson.core:jackson-databind&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.yarn.user.classpath.first&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;true&quot;</span><span class="token plain">,</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">      </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.sql.parquet.enableVectorizedReader&quot;</span><span class="token builtin class-name" style="color:rgb(174, 129, 255)">:</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;false&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">  </span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token punctuation" style="color:rgb(249, 38, 114)">}</span></div></pre></div><div role="tabpanel" class="tabItem_Ymn6" hidden="">SynapseML can be conveniently installed on existing Spark clusters via the --packages option, examples:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">spark-shell --packages com.microsoft.azure:synapseml_2.12:1.0.4 </span><span class="token comment" style="color:rgb(136, 132, 111)"># Please use 1.0.4 version for Spark3.4 and 0.11.4-spark3.3 version for Spark3.3</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">pyspark --packages com.microsoft.azure:synapseml_2.12:1.0.4</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">spark-submit --packages com.microsoft.azure:synapseml_2.12:1.0.4 MyApp.jar </span></div></pre>This can be used in other Spark contexts too. For example, you can use SynapseML in<!-- --> <a href="https://github.com/Azure/aztk/">AZTK</a> by adding it to the<!-- --> <a href="https://github.com/Azure/aztk/wiki/PySpark-on-Azure-with-AZTK#optional-set-up-mmlspark">.aztk/spark-defaults.conf file</a>.</div><div role="tabpanel" class="tabItem_Ymn6" hidden=""><p>To install SynapseML on the<!-- --> <a href="http://community.cloud.databricks.com">Databricks cloud</a>, create a new<!-- --> <a href="https://docs.databricks.com/user-guide/libraries.html#libraries-from-maven-pypi-or-spark-packages">library from Maven coordinates</a> <!-- -->in your workspace. in your workspace.</p><p></p><p>For the coordinates:</p>Spark 3.4 Cluster:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">com.microsoft.azure:synapseml_2.12:1.0.4</span></div></pre>Spark 3.3 Cluster:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">com.microsoft.azure:synapseml_2.12:0.11.4-spark3.3</span></div></pre>with the resolver:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">https://mmlspark.azureedge.net/maven</span></div></pre>Ensure this library is attached to your target cluster(s).<p></p><p>Finally, ensure that your Spark cluster has at least Spark 3.4 and Scala 2.12.</p>You can use SynapseML in both your Scala and PySpark notebooks. To get started with our example notebooks import the following databricks archive:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">https://mmlspark.blob.core.windows.net/dbcs/SynapseMLExamplesv1.0.4.dbc</span></div></pre></div><div role="tabpanel" class="tabItem_Ymn6" hidden="">The easiest way to evaluate SynapseML is via our pre-built Docker container. To do so, run the following command:<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token function" style="color:rgb(166, 226, 46)">docker</span><span class="token plain"> run -it -p </span><span class="token number" style="color:rgb(174, 129, 255)">8888</span><span class="token plain">:8888 -e </span><span class="token assign-left variable" style="color:rgb(248, 248, 242)">ACCEPT_EULA</span><span class="token operator">=</span><span class="token plain">yes mcr.microsoft.com/mmlspark/release</span></div></pre><p>Navigate to<!-- --> <a href="http://localhost:8888">http://localhost:8888</a> in your web browser to run the sample notebooks. See the<!-- --> <a href="https://github.com/microsoft/SynapseML/blob/master/docs/docker.md">documentation</a> <!-- -->for more on Docker use.</p>To read the EULA for using the docker image, run<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token function" style="color:rgb(166, 226, 46)">docker</span><span class="token plain"> run -it -p </span><span class="token number" style="color:rgb(174, 129, 255)">8888</span><span class="token plain">:8888 mcr.microsoft.com/mmlspark/release eula</span></div></pre></div><div role="tabpanel" class="tabItem_Ymn6" hidden="">To try out SynapseML on a Python (or Conda) installation you can get Spark installed via pip with<pre class="prism-code language-bash codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">pip </span><span class="token function" style="color:rgb(166, 226, 46)">install</span><span class="token plain"> pyspark</span></div></pre>You can then use pyspark as in the above example, or from python:<pre class="prism-code language-python codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token keyword">import</span><span class="token plain"> pyspark</span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">spark </span><span class="token operator">=</span><span class="token plain"> </span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token plain">pyspark</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">sql</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">SparkSession</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">builder</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">appName</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;MyApp&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">config</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.packages&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;com.microsoft.azure:synapseml_2.12:1.0.4&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)"># Please use 1.0.4 version for Spark3.4 and 0.11.4-spark3.3 version for Spark3.3</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">config</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token string" style="color:rgb(230, 219, 116)">&quot;spark.jars.repositories&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">,</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.azureedge.net/maven&quot;</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">        </span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">getOrCreate</span><span class="token punctuation" style="color:rgb(249, 38, 114)">(</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token punctuation" style="color:rgb(249, 38, 114)">)</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain"></span><span class="token keyword">import</span><span class="token plain"> synapse</span><span class="token punctuation" style="color:rgb(249, 38, 114)">.</span><span class="token plain">ml</span></div></pre></div><div role="tabpanel" class="tabItem_Ymn6" hidden="">If you are building a Spark application in Scala, add the following lines to your build.sbt:<pre class="prism-code language-jsx codeSnippet_f3C9" style="color:#f8f8f2;background-color:#272822"><div class="token-line" style="color:#f8f8f2"><span class="token plain">resolvers </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;SynapseML&quot;</span><span class="token plain"> at </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;https://mmlspark.azureedge.net/maven&quot;</span><span class="token plain"></span></div><div class="token-line" style="color:#f8f8f2"><span class="token plain">libraryDependencies </span><span class="token operator">+=</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;com.microsoft.azure&quot;</span><span class="token plain"> </span><span class="token operator">%</span><span class="token operator">%</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;synapseml_2.12&quot;</span><span class="token plain"> </span><span class="token operator">%</span><span class="token plain"> </span><span class="token string" style="color:rgb(230, 219, 116)">&quot;1.0.4&quot;</span><span class="token plain"> </span><span class="token comment" style="color:rgb(136, 132, 111)">// Please use 1.0.4 version for Spark3.2 and 1.0.4-spark3.3 version for Spark3.3</span></div></pre></div></div></div></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/search/index.html b/search/index.html
index 370b2e5823..06af4db9fb 100644
--- a/search/index.html
+++ b/search/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a class="navbar__item navbar__link" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/search/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><div class="container margin-vert--lg"><h1>Search the documentation</h1><form class="row"><div class="col searchQueryColumn_RTkw col--9"><input type="search" name="q" class="searchQueryInput_u2C7" placeholder="Type your search here" aria-label="Search" autocomplete="off" autofocus=""></div><div class="col col--3 padding-left--none searchVersionColumn_ypXd"><select class="searchVersionInput_m0Ui"><option selected="" label="Next" value="current"></option><option label="1.0.4" value="1.0.4"></option><option label="1.0.3" value="1.0.3"></option><option label="1.0.2" value="1.0.2"></option><option label="1.0.1" value="1.0.1"></option><option label="0.11.4" value="0.11.4"></option><option label="0.11.3" value="0.11.3"></option></select></div></form><div class="row"><div class="col col--8 searchResultsColumn_JPFH"></div><div class="col col--4 text--right searchLogoColumn_rJIA"><a target="_blank" rel="noopener noreferrer" href="https://www.algolia.com/" aria-label="Search by Algolia"><svg viewBox="0 0 168 24" class="algoliaLogo_rT1R"><g fill="none"><path class="algoliaLogoPathFill_WdUC" d="M120.925 18.804c-4.386.02-4.386-3.54-4.386-4.106l-.007-13.336 2.675-.424v13.254c0 .322 0 2.358 1.718 2.364v2.248zm-10.846-2.18c.821 0 1.43-.047 1.855-.129v-2.719a6.334 6.334 0 0 0-1.574-.199 5.7 5.7 0 0 0-.897.069 2.699 2.699 0 0 0-.814.24c-.24.116-.439.28-.582.491-.15.212-.219.335-.219.656 0 .628.219.991.616 1.23s.938.362 1.615.362zm-.233-9.7c.883 0 1.629.109 2.231.328.602.218 1.088.525 1.444.915.363.396.609.922.76 1.483.157.56.232 1.175.232 1.85v6.874a32.5 32.5 0 0 1-1.868.314c-.834.123-1.772.185-2.813.185-.69 0-1.327-.069-1.895-.198a4.001 4.001 0 0 1-1.471-.636 3.085 3.085 0 0 1-.951-1.134c-.226-.465-.343-1.12-.343-1.803 0-.656.13-1.073.384-1.525a3.24 3.24 0 0 1 1.047-1.106c.445-.287.95-.492 1.532-.615a8.8 8.8 0 0 1 1.82-.185 8.404 8.404 0 0 1 1.972.24v-.438c0-.307-.035-.6-.11-.874a1.88 1.88 0 0 0-.384-.73 1.784 1.784 0 0 0-.724-.493 3.164 3.164 0 0 0-1.143-.205c-.616 0-1.177.075-1.69.164a7.735 7.735 0 0 0-1.26.307l-.321-2.192c.335-.117.834-.233 1.478-.349a10.98 10.98 0 0 1 2.073-.178zm52.842 9.626c.822 0 1.43-.048 1.854-.13V13.7a6.347 6.347 0 0 0-1.574-.199c-.294 0-.595.021-.896.069a2.7 2.7 0 0 0-.814.24 1.46 1.46 0 0 0-.582.491c-.15.212-.218.335-.218.656 0 .628.218.991.615 1.23.404.245.938.362 1.615.362zm-.226-9.694c.883 0 1.629.108 2.231.327.602.219 1.088.526 1.444.915.355.39.609.923.759 1.483a6.8 6.8 0 0 1 .233 1.852v6.873c-.41.088-1.034.19-1.868.314-.834.123-1.772.184-2.813.184-.69 0-1.327-.068-1.895-.198a4.001 4.001 0 0 1-1.471-.635 3.085 3.085 0 0 1-.951-1.134c-.226-.465-.343-1.12-.343-1.804 0-.656.13-1.073.384-1.524.26-.45.608-.82 1.047-1.107.445-.286.95-.491 1.532-.614a8.803 8.803 0 0 1 2.751-.13c.329.034.671.096 1.04.185v-.437a3.3 3.3 0 0 0-.109-.875 1.873 1.873 0 0 0-.384-.731 1.784 1.784 0 0 0-.724-.492 3.165 3.165 0 0 0-1.143-.205c-.616 0-1.177.075-1.69.164a7.75 7.75 0 0 0-1.26.307l-.321-2.193c.335-.116.834-.232 1.478-.348a11.633 11.633 0 0 1 2.073-.177zm-8.034-1.271a1.626 1.626 0 0 1-1.628-1.62c0-.895.725-1.62 1.628-1.62.904 0 1.63.725 1.63 1.62 0 .895-.733 1.62-1.63 1.62zm1.348 13.22h-2.689V7.27l2.69-.423v11.956zm-4.714 0c-4.386.02-4.386-3.54-4.386-4.107l-.008-13.336 2.676-.424v13.254c0 .322 0 2.358 1.718 2.364v2.248zm-8.698-5.903c0-1.156-.253-2.119-.746-2.788-.493-.677-1.183-1.01-2.067-1.01-.882 0-1.574.333-2.065 1.01-.493.676-.733 1.632-.733 2.788 0 1.168.246 1.953.74 2.63.492.683 1.183 1.018 2.066 1.018.882 0 1.574-.342 2.067-1.019.492-.683.738-1.46.738-2.63zm2.737-.007c0 .902-.13 1.584-.397 2.33a5.52 5.52 0 0 1-1.128 1.906 4.986 4.986 0 0 1-1.752 1.223c-.685.286-1.739.45-2.265.45-.528-.006-1.574-.157-2.252-.45a5.096 5.096 0 0 1-1.744-1.223c-.487-.527-.863-1.162-1.137-1.906a6.345 6.345 0 0 1-.41-2.33c0-.902.123-1.77.397-2.508a5.554 5.554 0 0 1 1.15-1.892 5.133 5.133 0 0 1 1.75-1.216c.679-.287 1.425-.423 2.232-.423.808 0 1.553.142 2.237.423a4.88 4.88 0 0 1 1.753 1.216 5.644 5.644 0 0 1 1.135 1.892c.287.738.431 1.606.431 2.508zm-20.138 0c0 1.12.246 2.363.738 2.882.493.52 1.13.78 1.91.78.424 0 .828-.062 1.204-.178.377-.116.677-.253.917-.417V9.33a10.476 10.476 0 0 0-1.766-.226c-.971-.028-1.71.37-2.23 1.004-.513.636-.773 1.75-.773 2.788zm7.438 5.274c0 1.824-.466 3.156-1.404 4.004-.936.846-2.367 1.27-4.296 1.27-.705 0-2.17-.137-3.34-.396l.431-2.118c.98.205 2.272.26 2.95.26 1.074 0 1.84-.219 2.299-.656.459-.437.684-1.086.684-1.948v-.437a8.07 8.07 0 0 1-1.047.397c-.43.13-.93.198-1.492.198-.739 0-1.41-.116-2.018-.349a4.206 4.206 0 0 1-1.567-1.025c-.431-.45-.774-1.017-1.013-1.694-.24-.677-.363-1.885-.363-2.773 0-.834.13-1.88.384-2.577.26-.696.629-1.298 1.129-1.796.493-.498 1.095-.881 1.8-1.162a6.605 6.605 0 0 1 2.428-.457c.87 0 1.67.109 2.45.24.78.129 1.444.265 1.985.415V18.17zM6.972 6.677v1.627c-.712-.446-1.52-.67-2.425-.67-.585 0-1.045.13-1.38.391a1.24 1.24 0 0 0-.502 1.03c0 .425.164.765.494 1.02.33.256.835.532 1.516.83.447.192.795.356 1.045.495.25.138.537.332.862.582.324.25.563.548.718.894.154.345.23.741.23 1.188 0 .947-.334 1.691-1.004 2.234-.67.542-1.537.814-2.601.814-1.18 0-2.16-.229-2.936-.686v-1.708c.84.628 1.814.942 2.92.942.585 0 1.048-.136 1.388-.407.34-.271.51-.646.51-1.125 0-.287-.1-.55-.302-.79-.203-.24-.42-.42-.655-.542-.234-.123-.585-.29-1.053-.503a61.27 61.27 0 0 1-.582-.271 13.67 13.67 0 0 1-.55-.287 4.275 4.275 0 0 1-.567-.351 6.92 6.92 0 0 1-.455-.4c-.18-.17-.31-.34-.39-.51-.08-.17-.155-.37-.224-.598a2.553 2.553 0 0 1-.104-.742c0-.915.333-1.638.998-2.17.664-.532 1.523-.798 2.576-.798.968 0 1.793.17 2.473.51zm7.468 5.696v-.287c-.022-.607-.187-1.088-.495-1.444-.309-.357-.75-.535-1.324-.535-.532 0-.99.194-1.373.583-.382.388-.622.949-.717 1.683h3.909zm1.005 2.792v1.404c-.596.34-1.383.51-2.362.51-1.255 0-2.255-.377-3-1.132-.744-.755-1.116-1.744-1.116-2.968 0-1.297.34-2.316 1.021-3.055.68-.74 1.548-1.11 2.6-1.11 1.033 0 1.852.323 2.458.966.606.644.91 1.572.91 2.784 0 .33-.033.676-.096 1.038h-5.314c.107.702.405 1.239.894 1.611.49.372 1.106.558 1.85.558.862 0 1.58-.202 2.155-.606zm6.605-1.77h-1.212c-.596 0-1.045.116-1.349.35-.303.234-.454.532-.454.894 0 .372.117.664.35.877.235.213.575.32 1.022.32.51 0 .912-.142 1.204-.424.293-.281.44-.651.44-1.108v-.91zm-4.068-2.554V9.325c.627-.361 1.457-.542 2.489-.542 2.116 0 3.175 1.026 3.175 3.08V17h-1.548v-.957c-.415.68-1.143 1.02-2.186 1.02-.766 0-1.38-.22-1.843-.661-.462-.442-.694-1.003-.694-1.684 0-.776.293-1.38.878-1.81.585-.431 1.404-.647 2.457-.647h1.34V11.8c0-.554-.133-.971-.399-1.253-.266-.282-.707-.423-1.324-.423a4.07 4.07 0 0 0-2.345.718zm9.333-1.93v1.42c.394-1 1.101-1.5 2.123-1.5.148 0 .313.016.494.048v1.531a1.885 1.885 0 0 0-.75-.143c-.542 0-.989.24-1.34.718-.351.479-.527 1.048-.527 1.707V17h-1.563V8.91h1.563zm5.01 4.084c.022.82.272 1.492.75 2.019.479.526 1.15.79 2.01.79.639 0 1.235-.176 1.788-.527v1.404c-.521.319-1.186.479-1.995.479-1.265 0-2.276-.4-3.031-1.197-.755-.798-1.133-1.792-1.133-2.984 0-1.16.38-2.151 1.14-2.975.761-.825 1.79-1.237 3.088-1.237.702 0 1.346.149 1.93.447v1.436a3.242 3.242 0 0 0-1.77-.495c-.84 0-1.513.266-2.019.798-.505.532-.758 1.213-.758 2.042zM40.24 5.72v4.579c.458-1 1.293-1.5 2.505-1.5.787 0 1.42.245 1.899.734.479.49.718 1.17.718 2.042V17h-1.564v-5.106c0-.553-.14-.98-.422-1.284-.282-.303-.652-.455-1.11-.455-.531 0-1.002.202-1.411.606-.41.405-.615 1.022-.615 1.851V17h-1.563V5.72h1.563zm14.966 10.02c.596 0 1.096-.253 1.5-.758.404-.506.606-1.157.606-1.955 0-.915-.202-1.62-.606-2.114-.404-.495-.92-.742-1.548-.742-.553 0-1.05.224-1.491.67-.442.447-.662 1.133-.662 2.058 0 .958.212 1.67.638 2.138.425.469.946.703 1.563.703zM53.004 5.72v4.42c.574-.894 1.388-1.341 2.44-1.341 1.022 0 1.857.383 2.506 1.149.649.766.973 1.781.973 3.047 0 1.138-.309 2.109-.925 2.912-.617.803-1.463 1.205-2.537 1.205-1.075 0-1.894-.447-2.457-1.34V17h-1.58V5.72h1.58zm9.908 11.104l-3.223-7.913h1.739l1.005 2.632 1.26 3.415c.096-.32.48-1.458 1.15-3.415l.909-2.632h1.66l-2.92 7.866c-.777 2.074-1.963 3.11-3.559 3.11a2.92 2.92 0 0 1-.734-.079v-1.34c.17.042.351.064.543.064 1.032 0 1.755-.57 2.17-1.708z"></path><path fill="#5468FF" d="M78.988.938h16.594a2.968 2.968 0 0 1 2.966 2.966V20.5a2.967 2.967 0 0 1-2.966 2.964H78.988a2.967 2.967 0 0 1-2.966-2.964V3.897A2.961 2.961 0 0 1 78.988.938z"></path><path fill="white" d="M89.632 5.967v-.772a.978.978 0 0 0-.978-.977h-2.28a.978.978 0 0 0-.978.977v.793c0 .088.082.15.171.13a7.127 7.127 0 0 1 1.984-.28c.65 0 1.295.088 1.917.259.082.02.164-.04.164-.13m-6.248 1.01l-.39-.389a.977.977 0 0 0-1.382 0l-.465.465a.973.973 0 0 0 0 1.38l.383.383c.062.061.15.047.205-.014.226-.307.472-.601.746-.874.281-.28.568-.526.883-.751.068-.042.075-.137.02-.2m4.16 2.453v3.341c0 .096.104.165.192.117l2.97-1.537c.068-.034.089-.117.055-.184a3.695 3.695 0 0 0-3.08-1.866c-.068 0-.136.054-.136.13m0 8.048a4.489 4.489 0 0 1-4.49-4.482 4.488 4.488 0 0 1 4.49-4.482 4.488 4.488 0 0 1 4.489 4.482 4.484 4.484 0 0 1-4.49 4.482m0-10.85a6.363 6.363 0 1 0 0 12.729 6.37 6.37 0 0 0 6.372-6.368 6.358 6.358 0 0 0-6.371-6.36"></path></g></svg></a></div></div></div></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file
diff --git a/sitemap.xml b/sitemap.xml
index 6643f2157c..a811e67b33 100644
--- a/sitemap.xml
+++ b/sitemap.xml
@@ -1 +1 @@
-<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://microsoft.github.io/SynapseML/blog/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2018/04/01/Flexible%20and%20Scalable%20Deep%20Learning%20with%20MMLSpark/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/06/01/MMLSpark%20Unifying%20Machine%20Learning%20Ecosystems%20at%20Massive%20Scales/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/08/24/Welcome%20to%20Azure%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/10/02/MMLSpark%20empowering%20AI%20for%20Good%20with%20Mark%20Hamilton/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2020/12/01/Large-Scale%20Intelligent%20Microservices/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/archive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/videos/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Creare%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Creare%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Synthetic%20difference%20in%20differences/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Synthetic%20difference%20in%20differences/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url></urlset>
\ No newline at end of file
+<?xml version="1.0" encoding="UTF-8"?><urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1"><url><loc>https://microsoft.github.io/SynapseML/blog/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2018/04/01/Flexible%20and%20Scalable%20Deep%20Learning%20with%20MMLSpark/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/06/01/MMLSpark%20Unifying%20Machine%20Learning%20Ecosystems%20at%20Massive%20Scales/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/08/24/Welcome%20to%20Azure%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2019/10/02/MMLSpark%20empowering%20AI%20for%20Good%20with%20Mark%20Hamilton/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/2020/12/01/Large-Scale%20Intelligent%20Microservices/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/archive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/blog/overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/videos/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Creare%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.3/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Creare%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/0.11.4/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.1/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.2/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/1.0.3/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Synthetic%20difference%20in%20differences/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20Custom%20Embeddings%20and%20Approximate%20KNN%20on%20GPU/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/next/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Deploy%20Models/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Deploy%20Models/Quickstart%20-%20Deploying%20a%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Advanced%20Usage%20-%20Async,%20Batching,%20and%20Multi-Key/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Geospatial%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Multivariate%20Anomaly%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Celebrity%20Quotes/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Analyze%20Text/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20a%20Visual%20Search%20Engine/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Create%20Audiobooks/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Document%20Question%20and%20Answering%20with%20PDFs/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Flooding%20Risk/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/AI%20Services/Quickstart%20-%20Predictive%20Maintenance/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Anomaly%20Detection/Quickstart%20-%20Isolation%20Forests/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Causal%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Measure%20Heterogeneous%20Effects/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Causal%20Inference/Quickstart%20-%20Synthetic%20difference%20in%20differences/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20SparkML%20vs%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Train%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Tabular%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Classification/Quickstart%20-%20Vowpal%20Wabbit%20on%20Text%20Data/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Distributed%20Training/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Getting%20Started/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/ONNX/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Text%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Fine-tune%20a%20Vision%20Classifier/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20ONNX%20Model%20Inference/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Deep%20Learning/Quickstart%20-%20Transfer%20Learn%20for%20Image%20Classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Hyperparameter%20Tuning/HyperOpt/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Hyperparameter%20Tuning/Quickstart%20-%20Random%20Search/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/LightGBM/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/LightGBM/Quickstart%20-%20Classification,%20Ranking,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Langchain/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding%20and%20GPU%20based%20KNN/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20OpenAI%20Embedding/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenAI/Quickstart%20-%20Understand%20and%20Search%20Forms/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/OpenCV/Image%20Transformations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Cyber%20ML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Anomalous%20Access%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Quickstart%20-%20Exploring%20Art%20Across%20Cultures/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Other%20Algorithms/Smart%20Adaptive%20Recommendations/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Data%20Cleaning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Train%20Regressor/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Regression/Quickstart%20-%20Vowpal%20Wabbit%20and%20LightGBM/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Explanation%20Dashboard/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Image%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Interpreting%20Model%20Predictions/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/PDP%20and%20ICE%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Data%20Balance%20Analysis/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Quickstart%20-%20Snow%20Leopard%20Detection/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Tabular%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Responsible%20AI/Text%20Explainers/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Contextual%20Bandits/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Multi-class%20classification/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20SparkML%20Vectors/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification%20using%20VW-native%20Format/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Explore%20Algorithms/Vowpal%20Wabbit/Quickstart%20-%20Classification,%20Quantile%20Regression,%20and%20Regression/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Create%20a%20Spark%20Cluster/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Install%20SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Quickstart%20-%20Your%20First%20Models/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Get%20Started/Set%20up%20Cognitive%20Services/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_causal/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_lightgbm/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/estimators/estimators_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_cognitive/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_core/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_deep_learning/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_opencv/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Quick%20Examples/transformers/transformers_vw/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Contributor%20Guide/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Developer%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Docker%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Dotnet%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/Quickstart%20-%20LightGBM%20in%20Dotnet/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Reference/R%20Setup/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Autologging/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Install/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/docs/Use%20with%20MLFlow/Overview/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url><url><loc>https://microsoft.github.io/SynapseML/</loc><changefreq>weekly</changefreq><priority>0.5</priority></url></urlset>
\ No newline at end of file
diff --git a/videos/index.html b/videos/index.html
index fd6a1abe6a..2b12273612 100644
--- a/videos/index.html
+++ b/videos/index.html
@@ -16,13 +16,13 @@
 <link rel="search" type="application/opensearchdescription+xml" title="SynapseML" href="/SynapseML/opensearch.xml">
 
 <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.13.11/dist/katex.min.css" integrity="sha384-Um5gpz1odJg5Z4HAmzPtgZKdTBHZdw8S29IecapCSB31ligYPhHQZMIlWLYQGVoc" crossorigin="anonymous"><link rel="stylesheet" href="/SynapseML/assets/css/styles.0b62a553.css">
-<link rel="preload" href="/SynapseML/assets/js/runtime~main.ea772462.js" as="script">
-<link rel="preload" href="/SynapseML/assets/js/main.f50c121d.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/runtime~main.6b063ea3.js" as="script">
+<link rel="preload" href="/SynapseML/assets/js/main.acd258f6.js" as="script">
 </head>
 <body class="navigation-with-keyboard">
 <script>!function(){function t(t){document.documentElement.setAttribute("data-theme",t)}var e=function(){var t=null;try{t=new URLSearchParams(window.location.search).get("docusaurus-theme")}catch(t){}return t}()||function(){var t=null;try{t=localStorage.getItem("theme")}catch(t){}return t}();t(null!==e?e:"dark")}(),document.documentElement.setAttribute("data-announcement-bar-initially-dismissed",function(){try{return"true"===localStorage.getItem("docusaurus.announcement.dismiss")}catch(t){}return!1}())</script><div id="__docusaurus">
 <div role="region" aria-label="Skip to main content"><a class="skipToContent_fXgn" href="#__docusaurus_skipToContent_fallback">Skip to main content</a></div><div class="announcementBar_mb4j" role="banner"><div class="announcementBarPlaceholder_vyr4"></div><div class="content_knG7 announcementBarContent_xLdY">⭐️ If you like SynapseML, consider giving it a star on <a target="_blank" rel="noopener noreferrer" href="https://github.com/Microsoft/SynapseML">GitHub</a> ⭐</div><button type="button" aria-label="Close" class="clean-btn close closeButton_CVFx announcementBarClose_gvF7"><svg viewBox="0 0 15 15" width="14" height="14"><g stroke="currentColor" stroke-width="3.1"><path d="M.75.75l13.5 13.5M14.25.75L.75 14.25"></path></g></svg></button></div><nav aria-label="Main" class="navbar navbar--fixed-top"><div class="navbar__inner"><div class="navbar__items"><button aria-label="Toggle navigation bar" aria-expanded="false" class="navbar__toggle clean-btn" type="button"><svg width="30" height="30" viewBox="0 0 30 30" aria-hidden="true"><path stroke="currentColor" stroke-linecap="round" stroke-miterlimit="10" stroke-width="2" d="M4 7h22M4 15h22M4 23h22"></path></svg></button><a class="navbar__brand" href="/SynapseML/"><div class="navbar__logo"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--light_HNdA"><img src="/SynapseML/img/logo.svg" alt="SynapseML Logo" class="themedImage_ToTc themedImage--dark_i4oU"></div><b class="navbar__title text--truncate">SynapseML</b></a><a class="navbar__item navbar__link" href="/SynapseML/docs/Overview/">Docs</a><a class="navbar__item navbar__link" href="/SynapseML/blog/">Blog</a><a aria-current="page" class="navbar__item navbar__link navbar__link--active" href="/SynapseML/videos/">Videos</a></div><div class="navbar__items navbar__items--right"><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a class="navbar__link" aria-haspopup="true" aria-expanded="false" role="button" href="/SynapseML/docs/Overview/">1.0.4</a><ul class="dropdown__menu"><li><a class="dropdown__link" href="/SynapseML/docs/next/Overview/">Next</a></li><li><a class="dropdown__link" href="/SynapseML/docs/Overview/">1.0.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.3/Overview/">1.0.3</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.2/Overview/">1.0.2</a></li><li><a class="dropdown__link" href="/SynapseML/docs/1.0.1/Overview/">1.0.1</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.4/Overview/">0.11.4</a></li><li><a class="dropdown__link" href="/SynapseML/docs/0.11.3/Overview/">0.11.3</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link"><svg viewBox="0 0 24 24" width="20" height="20" aria-hidden="true" class="iconLanguage_nlXk"><path fill="currentColor" d="M12.87 15.07l-2.54-2.51.03-.03c1.74-1.94 2.98-4.17 3.71-6.53H17V4h-7V2H8v2H1v1.99h11.17C11.5 7.92 10.44 9.75 9 11.35 8.07 10.32 7.3 9.19 6.69 8h-2c.73 1.63 1.73 3.17 2.98 4.56l-5.09 5.02L4 19l5-5 3.11 3.11.76-2.04zM18.5 10h-2L12 22h2l1.12-3h4.75L21 22h2l-4.5-12zm-2.62 7l1.62-4.33L19.12 17h-3.24z"></path></svg>English</a><ul class="dropdown__menu"><li><a href="/SynapseML/videos/" target="_self" rel="noopener noreferrer" class="dropdown__link dropdown__link--active" lang="en">English</a></li></ul></div><div class="navbar__item dropdown dropdown--hoverable dropdown--right"><a href="#" aria-haspopup="true" aria-expanded="false" role="button" class="navbar__link">Developer Docs</a><ul class="dropdown__menu"><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Python<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li><li><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/com/microsoft/azure/synapse/ml/index.html" target="_blank" rel="noopener noreferrer" class="dropdown__link">Scala<svg width="12" height="12" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="navbar__item navbar__link header-github-link" aria-label="GitHub repository"></a><div class="toggle_vylO colorModeToggle_DEke"><button class="clean-btn toggleButton_gllP toggleButtonDisabled_aARS" type="button" disabled="" title="Switch between dark and light mode (currently dark mode)" aria-label="Switch between dark and light mode (currently dark mode)" aria-live="polite"><svg viewBox="0 0 24 24" width="24" height="24" class="lightToggleIcon_pyhR"><path fill="currentColor" d="M12,9c1.65,0,3,1.35,3,3s-1.35,3-3,3s-3-1.35-3-3S10.35,9,12,9 M12,7c-2.76,0-5,2.24-5,5s2.24,5,5,5s5-2.24,5-5 S14.76,7,12,7L12,7z M2,13l2,0c0.55,0,1-0.45,1-1s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S1.45,13,2,13z M20,13l2,0c0.55,0,1-0.45,1-1 s-0.45-1-1-1l-2,0c-0.55,0-1,0.45-1,1S19.45,13,20,13z M11,2v2c0,0.55,0.45,1,1,1s1-0.45,1-1V2c0-0.55-0.45-1-1-1S11,1.45,11,2z M11,20v2c0,0.55,0.45,1,1,1s1-0.45,1-1v-2c0-0.55-0.45-1-1-1C11.45,19,11,19.45,11,20z M5.99,4.58c-0.39-0.39-1.03-0.39-1.41,0 c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0s0.39-1.03,0-1.41L5.99,4.58z M18.36,16.95 c-0.39-0.39-1.03-0.39-1.41,0c-0.39,0.39-0.39,1.03,0,1.41l1.06,1.06c0.39,0.39,1.03,0.39,1.41,0c0.39-0.39,0.39-1.03,0-1.41 L18.36,16.95z M19.42,5.99c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06c-0.39,0.39-0.39,1.03,0,1.41 s1.03,0.39,1.41,0L19.42,5.99z M7.05,18.36c0.39-0.39,0.39-1.03,0-1.41c-0.39-0.39-1.03-0.39-1.41,0l-1.06,1.06 c-0.39,0.39-0.39,1.03,0,1.41s1.03,0.39,1.41,0L7.05,18.36z"></path></svg><svg viewBox="0 0 24 24" width="24" height="24" class="darkToggleIcon_wfgR"><path fill="currentColor" d="M9.37,5.51C9.19,6.15,9.1,6.82,9.1,7.5c0,4.08,3.32,7.4,7.4,7.4c0.68,0,1.35-0.09,1.99-0.27C17.45,17.19,14.93,19,12,19 c-3.86,0-7-3.14-7-7C5,9.07,6.81,6.55,9.37,5.51z M12,3c-4.97,0-9,4.03-9,9s4.03,9,9,9s9-4.03,9-9c0-0.46-0.04-0.92-0.1-1.36 c-0.98,1.37-2.58,2.26-4.4,2.26c-2.98,0-5.4-2.42-5.4-5.4c0-1.81,0.89-3.42,2.26-4.4C12.92,3.04,12.46,3,12,3L12,3z"></path></svg></button></div><div class="searchBox_ZlJk"><button type="button" class="DocSearch DocSearch-Button" aria-label="Search"><span class="DocSearch-Button-Container"><svg width="20" height="20" class="DocSearch-Search-Icon" viewBox="0 0 20 20"><path d="M14.386 14.386l4.0877 4.0877-4.0877-4.0877c-2.9418 2.9419-7.7115 2.9419-10.6533 0-2.9419-2.9418-2.9419-7.7115 0-10.6533 2.9418-2.9419 7.7115-2.9419 10.6533 0 2.9419 2.9418 2.9419 7.7115 0 10.6533z" stroke="currentColor" fill="none" fill-rule="evenodd" stroke-linecap="round" stroke-linejoin="round"></path></svg><span class="DocSearch-Button-Placeholder">Search</span></span><span class="DocSearch-Button-Keys"></span></button></div></div></div><div role="presentation" class="navbar-sidebar__backdrop"></div></nav><div id="__docusaurus_skipToContent_fallback" class="main-wrapper mainWrapper_z2l0"><header><div class="container"><div class="row"><div class="col col--6 col--offset-1"><h1 class="videosTitle_lBaH">SynapseML Videos</h1></div></div></div></header><main><div class="container margin-vert--lg"><div class="row margin-bottom--lg"><div class="col col--8 col--offset-2"><h1>Create Custom Search Engines from Unstructured Data with SynapseML</h1><p>Learn how to create custom search engines from unstructured data using Form Recognizer, Form Ontology Learner, Translation, GPT-3 and Azure Search in SynapseML.</p></div><div style="width:640px;height:360px" class="col col--8 col--offset-2"></div></div><div class="row margin-bottom--lg"><div class="col col--8 col--offset-2"><h1>Unsupervised Currency Detection</h1><h2>Spark + AI Summit Keynote 2019</h2><p>We use Bing on Spark, CNTK on Spark, Spark Serving, and ML Ops to help those with visual impairments work with currency.</p></div><div style="width:640px;height:360px" class="col col--8 col--offset-2"></div></div><div class="row margin-vert--lg"><div class="col col--8 col--offset-2"><h1>Unsupervised Fire Safety</h1><h2>Spark + AI Summit Europe Keynote 2018</h2><p>We use Bing on Spark, CNTK on Spark, and Spark serving to create a automated fire detection service for gas station safety. We then deploy this to an FPGA accelerated camera for Shell Industries.</p></div><div style="width:640px;height:360px" class="col col--8 col--offset-2"></div></div><div class="row margin-vert--lg"><div class="col col--8 col--offset-2"><h1>Predictive Maintenance with UAVs</h1><h2>Spark + AI Summit 2018</h2><p>We use CNTK on Spark to distribute a Faster RCNN object detection network and deploy it as a web service with SynapseML Serving for use on Unmanned Aerial Vehicals (UAVs)</p><a href="https://databricks.com/sparkaisummit/north-america/spark-summit-2018-keynotes#Intelligent-cloud" class="watchNowButton_wrG3">Watch Now</a></div></div><div class="row margin-vert--lg"><div class="col col--8 col--offset-2"><h1>Automated Snow Leopard Detection</h1><p>We have partnered with the Snow Leopard Trust to create an intelligent snow leopard identification system. This project helped eliminate thousands of hours of searching through photos.</p><div><a href="https://news.microsoft.com/transform/snow-leopard-selfies-ai-save-species/" class="watchNowButton_wrG3">Read More on Microsoft Transform</a><a href="https://www.geekwire.com/2018/microsoft-says-ai-finally-ready-broader-use-help-solve-earths-environmental-woes/" class="watchNowButton_wrG3">Read More on Geekwire</a></div></div></div><div class="row margin-vert--lg"><div class="col col--8 col--offset-2"><h1>Real-time Intelligent Analytics</h1><h2>Microsoft Connect Keynote 2017</h2><p>We use CNTK on Spark and deep transfer learning to create a real-time geospacial application for conservation biology in 5 minutes</p><a href="https://channel9.msdn.com/Events/Connect/2017/G102" class="watchNowButton_wrG3">Watch Now</a></div></div></div></main></div><footer class="footer footer--dark"><div class="container container-fluid"><div class="row footer__links"><div class="col footer__col"><div class="footer__title">Docs</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Install SynapseML/">Installation</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/docs/Get Started/Quickstart - Your First Models/">Getting Started</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/pyspark/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Python API Reference</a></li><li class="footer__item"><a href="https://mmlspark.blob.core.windows.net/docs/1.0.4/scala/index.html" target="_blank" rel="noopener noreferrer" class="footer__link-item">Scala API Reference</a></li></ul></div><div class="col footer__col"><div class="footer__title">More</div><ul class="footer__items clean-list"><li class="footer__item"><a class="footer__link-item" href="/SynapseML/blog/">Blog</a></li><li class="footer__item"><a class="footer__link-item" href="/SynapseML/videos/">Videos</a></li></ul></div><div class="col footer__col"><div class="footer__title">Community</div><ul class="footer__items clean-list"><li class="footer__item"><a href="https://github.com/microsoft/SynapseML" target="_blank" rel="noopener noreferrer" class="footer__link-item">GitHub<svg width="13.5" height="13.5" aria-hidden="true" viewBox="0 0 24 24" class="iconExternalLink_nPIU"><path fill="currentColor" d="M21 13v10h-21v-19h12v2h-10v15h17v-8h2zm3-12h-10.988l4.035 4-6.977 7.07 2.828 2.828 6.977-7.07 4.125 4.172v-11z"></path></svg></a></li></ul></div></div><div class="footer__bottom text--center"><div class="footer__copyright">Copyright © 2024 Microsoft.</div></div></div></footer></div>
-<script src="/SynapseML/assets/js/runtime~main.ea772462.js"></script>
-<script src="/SynapseML/assets/js/main.f50c121d.js"></script>
+<script src="/SynapseML/assets/js/runtime~main.6b063ea3.js"></script>
+<script src="/SynapseML/assets/js/main.acd258f6.js"></script>
 </body>
 </html>
\ No newline at end of file